diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_0_8.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_0_8.json new file mode 100644 index 000000000000..513488eb6713 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_0_8.json @@ -0,0 +1,24263 @@ +{ + "GlobalRank": null, + "LocalRank": 0, + "Mode": "DynamicRange", + "Nodes": { + "language_model.model.layers.0.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 11.1875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.51171875 + ] + ] + } + }, + "language_model.model.layers.0.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.640625 + ] + ] + ], + "outputs": [ + [ + [ + 0.83203125 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.5234375 + ] + ], + [ + [ + 6.15625 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.640625 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.5234375 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.640625 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 6.15625 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.640625 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.125 + ] + ], + [ + [ + 6.15625 + ] + ], + [ + [ + 0.640625 + ] + ] + ], + "outputs": [ + [ + [ + 0.640625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.0.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 2.546875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.0.feed_forward.down_proj": { + "inputs": [ + [ + [ + 37.25 + ] + ] + ], + "outputs": [ + [ + [ + 26.75 + ] + ], + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.734375 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 10.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.91796875 + ] + ] + ], + "outputs": [ + [ + [ + 0.67578125 + ] + ], + [ + [ + 1.3984375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.69140625 + ] + ], + [ + [ + 9.4375 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0859375 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.69140625 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.7578125 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 9.4375 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.0859375 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.875 + ] + ], + [ + [ + 9.4375 + ] + ], + [ + [ + 1.0859375 + ] + ] + ], + "outputs": [ + [ + [ + 0.91796875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.1.feed_forward.router": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "outputs": [ + [ + [ + 1.640625 + ] + ], + [ + [ + 14.0625 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.90234375 + ] + ], + [ + [ + 15.1875 + ] + ], + [ + [ + 20.5 + ] + ], + [ + [ + 19.625 + ] + ], + [ + [ + 18.375 + ] + ], + [ + [ + 18.75 + ] + ], + [ + [ + 14.3125 + ] + ], + [ + [ + 15.4375 + ] + ], + [ + [ + 16.75 + ] + ], + [ + [ + 16.0 + ] + ], + [ + [ + 5.78125 + ] + ], + [ + [ + 15.1875 + ] + ], + [ + [ + 21.25 + ] + ], + [ + [ + 24.625 + ] + ] + ] + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11767578125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4765625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37109375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.310546875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.244140625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 6.5625 + ] + ] + ], + "outputs": [ + [ + [ + 0.400390625 + ] + ], + [ + [ + 1.604866734412731e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.365234375 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 9.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.6171875 + ] + ] + ], + "outputs": [ + [ + [ + 0.7734375 + ] + ], + [ + [ + 1.4921875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.72265625 + ] + ], + [ + [ + 10.9375 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.90234375 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.72265625 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.53125 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 10.9375 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.90234375 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.5 + ] + ], + [ + [ + 10.9375 + ] + ], + [ + [ + 0.90234375 + ] + ] + ], + "outputs": [ + [ + [ + 0.6171875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.2.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 6.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.2.feed_forward.down_proj": { + "inputs": [ + [ + [ + 5.625 + ] + ] + ], + "outputs": [ + [ + [ + 4.75 + ] + ], + [ + [ + 25.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.498046875 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 7.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.294921875 + ] + ] + ], + "outputs": [ + [ + [ + 0.77734375 + ] + ], + [ + [ + 2.40625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.419921875 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.03759765625 + ] + ], + [ + [ + 2.625 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.4609375 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.03759765625 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.291015625 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 2.625 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.4609375 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 1.15625 + ] + ], + [ + [ + 2.625 + ] + ], + [ + [ + 0.451171875 + ] + ] + ], + "outputs": [ + [ + [ + 0.294921875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.3.feed_forward.router": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.52734375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "outputs": [ + [ + [ + 2.9375 + ] + ], + [ + [ + 16.25 + ] + ], + [ + [ + 16.375 + ] + ], + [ + [ + 18.25 + ] + ], + [ + [ + 15.4375 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 6.34375 + ] + ], + [ + [ + 18.125 + ] + ], + [ + [ + 18.0 + ] + ], + [ + [ + 15.9375 + ] + ], + [ + [ + 16.75 + ] + ], + [ + [ + 37.0 + ] + ], + [ + [ + 16.75 + ] + ], + [ + [ + 16.25 + ] + ], + [ + [ + 16.875 + ] + ], + [ + [ + 14.0 + ] + ], + [ + [ + 20.75 + ] + ] + ] + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1025390625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10546875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1025390625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 5.9375 + ] + ] + ], + "outputs": [ + [ + [ + 0.66796875 + ] + ], + [ + [ + 1.521856386081038e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5234375 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 14.3125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.140625 + ] + ] + ], + "outputs": [ + [ + [ + 1.46875 + ] + ], + [ + [ + 2.328125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.984375 + ] + ], + [ + [ + 15.3125 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.015625 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.984375 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.125 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.3125 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.015625 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.5 + ] + ], + [ + [ + 15.3125 + ] + ], + [ + [ + 2.8125 + ] + ] + ], + "outputs": [ + [ + [ + 2.140625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.4.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 6.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.4.feed_forward.down_proj": { + "inputs": [ + [ + [ + 6.59375 + ] + ] + ], + "outputs": [ + [ + [ + 2.796875 + ] + ], + [ + [ + 5.1875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.58984375 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 18.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.234375 + ] + ] + ], + "outputs": [ + [ + [ + 1.4921875 + ] + ], + [ + [ + 2.40625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.97265625 + ] + ], + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.625 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.97265625 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.75 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.625 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.75 + ] + ], + [ + [ + 18.625 + ] + ], + [ + [ + 2.625 + ] + ] + ], + "outputs": [ + [ + [ + 1.234375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.5.feed_forward.router": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 3.28125 + ] + ], + [ + [ + 20.75 + ] + ], + [ + [ + 30.375 + ] + ], + [ + [ + 28.875 + ] + ], + [ + [ + 41.0 + ] + ], + [ + [ + 19.375 + ] + ], + [ + [ + 22.625 + ] + ], + [ + [ + 16.25 + ] + ], + [ + [ + 19.875 + ] + ], + [ + [ + 23.625 + ] + ], + [ + [ + 27.875 + ] + ], + [ + [ + 16.625 + ] + ], + [ + [ + 38.75 + ] + ], + [ + [ + 25.375 + ] + ], + [ + [ + 33.25 + ] + ], + [ + [ + 62.75 + ] + ], + [ + [ + 27.75 + ] + ] + ] + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10888671875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09326171875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1083984375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.333984375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08642578125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.47265625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 7.34375 + ] + ] + ], + "outputs": [ + [ + [ + 1.3515625 + ] + ], + [ + [ + 1.521856386081038e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.35546875 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 14.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.296875 + ] + ] + ], + "outputs": [ + [ + [ + 0.94140625 + ] + ], + [ + [ + 1.96875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.96484375 + ] + ], + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.515625 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.96484375 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.296875 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.515625 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.1875 + ] + ], + [ + [ + 17.5 + ] + ], + [ + [ + 2.515625 + ] + ] + ], + "outputs": [ + [ + [ + 1.8125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.6.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 7.71875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.6.feed_forward.down_proj": { + "inputs": [ + [ + [ + 10.0625 + ] + ] + ], + "outputs": [ + [ + [ + 2.25 + ] + ], + [ + [ + 11.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 7.59375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.671875 + ] + ] + ], + "outputs": [ + [ + [ + 2.1875 + ] + ], + [ + [ + 2.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37109375 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.26171875 + ] + ], + [ + [ + 4.6875 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.26171875 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.859375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 4.6875 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 3.078125 + ] + ], + [ + [ + 4.6875 + ] + ], + [ + [ + 1.375 + ] + ] + ], + "outputs": [ + [ + [ + 0.671875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.7.feed_forward.router": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "outputs": [ + [ + [ + 16.0 + ] + ], + [ + [ + 41.0 + ] + ], + [ + [ + 31.625 + ] + ], + [ + [ + 24.625 + ] + ], + [ + [ + 9.6875 + ] + ], + [ + [ + 34.25 + ] + ], + [ + [ + 22.0 + ] + ], + [ + [ + 26.25 + ] + ], + [ + [ + 48.0 + ] + ], + [ + [ + 24.875 + ] + ], + [ + [ + 37.25 + ] + ], + [ + [ + 24.375 + ] + ], + [ + [ + 13.3125 + ] + ], + [ + [ + 27.125 + ] + ], + [ + [ + 33.5 + ] + ], + [ + [ + 45.5 + ] + ], + [ + [ + 10.1875 + ] + ] + ] + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.111328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0859375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08544921875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.61328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 11.8125 + ] + ] + ], + "outputs": [ + [ + [ + 1.390625 + ] + ], + [ + [ + 1.521856386081038e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.21875 + ] + ] + ], + "outputs": [ + [ + [ + 2.265625 + ] + ], + [ + [ + 3.390625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.9296875 + ] + ], + [ + [ + 17.0 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.78125 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.9296875 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.140625 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.0 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.78125 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.1875 + ] + ], + [ + [ + 17.0 + ] + ], + [ + [ + 3.78125 + ] + ] + ], + "outputs": [ + [ + [ + 2.21875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.8.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 8.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.8.feed_forward.down_proj": { + "inputs": [ + [ + [ + 10.875 + ] + ] + ], + "outputs": [ + [ + [ + 2.84375 + ] + ], + [ + [ + 9.8125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.48828125 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 16.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.46875 + ] + ] + ], + "outputs": [ + [ + [ + 1.609375 + ] + ], + [ + [ + 3.640625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.8125 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.15625 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.390625 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.8125 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.15625 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.0625 + ] + ], + [ + [ + 14.8125 + ] + ], + [ + [ + 3.109375 + ] + ] + ], + "outputs": [ + [ + [ + 2.46875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.9.feed_forward.router": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "outputs": [ + [ + [ + 3.875 + ] + ], + [ + [ + 42.0 + ] + ], + [ + [ + 37.75 + ] + ], + [ + [ + 33.25 + ] + ], + [ + [ + 40.75 + ] + ], + [ + [ + 32.5 + ] + ], + [ + [ + 56.5 + ] + ], + [ + [ + 38.0 + ] + ], + [ + [ + 42.5 + ] + ], + [ + [ + 48.25 + ] + ], + [ + [ + 36.75 + ] + ], + [ + [ + 52.0 + ] + ], + [ + [ + 36.75 + ] + ], + [ + [ + 64.0 + ] + ], + [ + [ + 47.25 + ] + ], + [ + [ + 53.0 + ] + ], + [ + [ + 30.875 + ] + ] + ] + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11376953125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09326171875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10546875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08984375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 10.875 + ] + ] + ], + "outputs": [ + [ + [ + 0.890625 + ] + ], + [ + [ + 1.604866734412731e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 17.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.765625 + ] + ] + ], + "outputs": [ + [ + [ + 1.984375 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.81640625 + ] + ], + [ + [ + 16.5 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.546875 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.81640625 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.171875 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.5 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.546875 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.375 + ] + ], + [ + [ + 15.75 + ] + ], + [ + [ + 3.546875 + ] + ] + ], + "outputs": [ + [ + [ + 2.765625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.10.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 9.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.10.feed_forward.down_proj": { + "inputs": [ + [ + [ + 15.375 + ] + ] + ], + "outputs": [ + [ + [ + 3.40625 + ] + ], + [ + [ + 12.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.408203125 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 15.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.734375 + ] + ] + ], + "outputs": [ + [ + [ + 8.625 + ] + ], + [ + [ + 8.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.375 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.68359375 + ] + ], + [ + [ + 8.75 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.46875 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.68359375 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.34375 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.46875 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 8.1875 + ] + ], + [ + [ + 8.375 + ] + ], + [ + [ + 5.46875 + ] + ] + ], + "outputs": [ + [ + [ + 3.625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.11.feed_forward.router": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "outputs": [ + [ + [ + 8.5625 + ] + ], + [ + [ + 47.5 + ] + ], + [ + [ + 48.5 + ] + ], + [ + [ + 70.5 + ] + ], + [ + [ + 27.5 + ] + ], + [ + [ + 30.75 + ] + ], + [ + [ + 48.0 + ] + ], + [ + [ + 12.8125 + ] + ], + [ + [ + 78.0 + ] + ], + [ + [ + 37.0 + ] + ], + [ + [ + 34.5 + ] + ], + [ + [ + 48.75 + ] + ], + [ + [ + 50.5 + ] + ], + [ + [ + 57.25 + ] + ], + [ + [ + 56.75 + ] + ], + [ + [ + 47.5 + ] + ], + [ + [ + 73.5 + ] + ] + ] + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.380859375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11279296875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.083984375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08837890625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 15.375 + ] + ] + ], + "outputs": [ + [ + [ + 3.75 + ] + ], + [ + [ + 1.521856386081038e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.244140625 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 20.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.15625 + ] + ] + ], + "outputs": [ + [ + [ + 1.796875 + ] + ], + [ + [ + 3.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.98828125 + ] + ], + [ + [ + 19.125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.453125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.98828125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.890625 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.453125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.4375 + ] + ], + [ + [ + 19.125 + ] + ], + [ + [ + 3.453125 + ] + ] + ], + "outputs": [ + [ + [ + 2.15625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.12.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 10.6875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.12.feed_forward.down_proj": { + "inputs": [ + [ + [ + 12.8125 + ] + ] + ], + "outputs": [ + [ + [ + 9.25 + ] + ], + [ + [ + 20.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.68359375 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.1875 + ] + ] + ], + "outputs": [ + [ + [ + 2.015625 + ] + ], + [ + [ + 6.34375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.8125 + ] + ], + [ + [ + 13.3125 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.640625 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.8125 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.1875 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.3125 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.640625 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.9375 + ] + ], + [ + [ + 13.3125 + ] + ], + [ + [ + 3.46875 + ] + ] + ], + "outputs": [ + [ + [ + 3.15625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.13.feed_forward.router": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "outputs": [ + [ + [ + 4.96875 + ] + ], + [ + [ + 57.75 + ] + ], + [ + [ + 51.0 + ] + ], + [ + [ + 20.625 + ] + ], + [ + [ + 60.75 + ] + ], + [ + [ + 52.0 + ] + ], + [ + [ + 59.0 + ] + ], + [ + [ + 25.0 + ] + ], + [ + [ + 54.75 + ] + ], + [ + [ + 42.0 + ] + ], + [ + [ + 27.25 + ] + ], + [ + [ + 51.0 + ] + ], + [ + [ + 58.5 + ] + ], + [ + [ + 48.75 + ] + ], + [ + [ + 30.875 + ] + ], + [ + [ + 60.25 + ] + ], + [ + [ + 32.5 + ] + ] + ] + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23828125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.46484375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09130859375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10888671875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10888671875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08642578125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11181640625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 20.25 + ] + ] + ], + "outputs": [ + [ + [ + 5.0625 + ] + ], + [ + [ + 3.135946492530624e+20 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 20.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.46875 + ] + ] + ], + "outputs": [ + [ + [ + 3.546875 + ] + ], + [ + [ + 5.71875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.85546875 + ] + ], + [ + [ + 15.6875 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.453125 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.85546875 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.734375 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.6875 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.453125 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.0625 + ] + ], + [ + [ + 15.6875 + ] + ], + [ + [ + 3.453125 + ] + ] + ], + "outputs": [ + [ + [ + 2.46875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.14.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 12.0625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.14.feed_forward.down_proj": { + "inputs": [ + [ + [ + 14.25 + ] + ] + ], + "outputs": [ + [ + [ + 5.375 + ] + ], + [ + [ + 20.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.470703125 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 23.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.4375 + ] + ] + ], + "outputs": [ + [ + [ + 20.0 + ] + ], + [ + [ + 21.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.046875 + ] + ], + [ + [ + 16.0 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.3125 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.046875 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.109375 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.0 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.3125 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.5 + ] + ], + [ + [ + 16.0 + ] + ], + [ + [ + 4.3125 + ] + ] + ], + "outputs": [ + [ + [ + 3.4375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.15.feed_forward.router": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 5.34375 + ] + ], + [ + [ + 64.5 + ] + ], + [ + [ + 63.0 + ] + ], + [ + [ + 55.5 + ] + ], + [ + [ + 31.875 + ] + ], + [ + [ + 50.25 + ] + ], + [ + [ + 71.5 + ] + ], + [ + [ + 60.25 + ] + ], + [ + [ + 50.5 + ] + ], + [ + [ + 74.0 + ] + ], + [ + [ + 91.0 + ] + ], + [ + [ + 33.5 + ] + ], + [ + [ + 58.75 + ] + ], + [ + [ + 82.5 + ] + ], + [ + [ + 44.75 + ] + ], + [ + [ + 75.0 + ] + ], + [ + [ + 32.75 + ] + ] + ] + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1025390625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10498046875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11572265625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.111328125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 22.25 + ] + ] + ], + "outputs": [ + [ + [ + 1.78125 + ] + ], + [ + [ + 1.604866734412731e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 23.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.8125 + ] + ] + ], + "outputs": [ + [ + [ + 2.4375 + ] + ], + [ + [ + 6.3125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.25 + ] + ], + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.03125 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.25 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.15625 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.03125 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.75 + ] + ], + [ + [ + 18.625 + ] + ], + [ + [ + 6.03125 + ] + ] + ], + "outputs": [ + [ + [ + 5.8125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.16.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 14.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.16.feed_forward.down_proj": { + "inputs": [ + [ + [ + 18.125 + ] + ] + ], + "outputs": [ + [ + [ + 6.65625 + ] + ], + [ + [ + 27.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5390625 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.671875 + ] + ] + ], + "outputs": [ + [ + [ + 19.5 + ] + ], + [ + [ + 27.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.171875 + ] + ], + [ + [ + 18.5 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.875 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.171875 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.484375 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.875 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.3125 + ] + ], + [ + [ + 18.5 + ] + ], + [ + [ + 4.875 + ] + ] + ], + "outputs": [ + [ + [ + 3.671875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.17.feed_forward.router": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 7.59375 + ] + ], + [ + [ + 60.25 + ] + ], + [ + [ + 77.0 + ] + ], + [ + [ + 67.5 + ] + ], + [ + [ + 106.5 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 53.5 + ] + ], + [ + [ + 61.25 + ] + ], + [ + [ + 54.5 + ] + ], + [ + [ + 97.5 + ] + ], + [ + [ + 74.0 + ] + ], + [ + [ + 54.5 + ] + ], + [ + [ + 65.5 + ] + ], + [ + [ + 76.5 + ] + ], + [ + [ + 37.75 + ] + ], + [ + [ + 93.0 + ] + ], + [ + [ + 75.0 + ] + ] + ] + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.240234375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.341796875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 28.875 + ] + ] + ], + "outputs": [ + [ + [ + 4.03125 + ] + ], + [ + [ + 1.521856386081038e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.53125 + ] + ] + ], + "outputs": [ + [ + [ + 6.625 + ] + ], + [ + [ + 12.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.09375 + ] + ], + [ + [ + 18.25 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.125 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.09375 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.375 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.25 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.125 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.0 + ] + ], + [ + [ + 18.125 + ] + ], + [ + [ + 5.9375 + ] + ] + ], + "outputs": [ + [ + [ + 4.53125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.18.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 20.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.44140625 + ] + ] + } + }, + "language_model.model.layers.18.feed_forward.down_proj": { + "inputs": [ + [ + [ + 18.25 + ] + ] + ], + "outputs": [ + [ + [ + 8.1875 + ] + ], + [ + [ + 354.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.490234375 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 44.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.78125 + ] + ] + ], + "outputs": [ + [ + [ + 31.5 + ] + ], + [ + [ + 37.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.046875 + ] + ], + [ + [ + 12.4375 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.625 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.046875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.859375 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 12.4375 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.625 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.5 + ] + ], + [ + [ + 12.4375 + ] + ], + [ + [ + 7.625 + ] + ] + ], + "outputs": [ + [ + [ + 5.78125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.19.feed_forward.router": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "outputs": [ + [ + [ + 7.46875 + ] + ], + [ + [ + 82.5 + ] + ], + [ + [ + 47.75 + ] + ], + [ + [ + 30.5 + ] + ], + [ + [ + 58.25 + ] + ], + [ + [ + 103.5 + ] + ], + [ + [ + 43.75 + ] + ], + [ + [ + 59.75 + ] + ], + [ + [ + 73.0 + ] + ], + [ + [ + 18.625 + ] + ], + [ + [ + 53.25 + ] + ], + [ + [ + 45.75 + ] + ], + [ + [ + 68.0 + ] + ], + [ + [ + 51.25 + ] + ], + [ + [ + 80.5 + ] + ], + [ + [ + 51.25 + ] + ], + [ + [ + 60.0 + ] + ] + ] + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3828125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23828125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.341796875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.43359375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 18.875 + ] + ] + ], + "outputs": [ + [ + [ + 4.78125 + ] + ], + [ + [ + 1.8354510353341004e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.498046875 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 53.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.6875 + ] + ] + ], + "outputs": [ + [ + [ + 12.8125 + ] + ], + [ + [ + 22.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1171875 + ] + ], + [ + [ + 20.375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.5 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1171875 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.53125 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.75 + ] + ], + [ + [ + 20.375 + ] + ], + [ + [ + 11.5 + ] + ] + ], + "outputs": [ + [ + [ + 4.6875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.20.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 42.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.20.feed_forward.down_proj": { + "inputs": [ + [ + [ + 26.625 + ] + ] + ], + "outputs": [ + [ + [ + 10.5 + ] + ], + [ + [ + 41.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.69140625 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 58.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.o_proj": { + "inputs": [ + [ + [ + 11.4375 + ] + ] + ], + "outputs": [ + [ + [ + 26.0 + ] + ], + [ + [ + 35.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.416015625 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.28125 + ] + ], + [ + [ + 18.375 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.0625 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.28125 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.6875 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.375 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 12.0625 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.375 + ] + ], + [ + [ + 18.375 + ] + ], + [ + [ + 12.0625 + ] + ] + ], + "outputs": [ + [ + [ + 11.4375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.21.feed_forward.router": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.45703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "outputs": [ + [ + [ + 10.375 + ] + ], + [ + [ + 44.75 + ] + ], + [ + [ + 70.5 + ] + ], + [ + [ + 42.5 + ] + ], + [ + [ + 39.25 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 58.25 + ] + ], + [ + [ + 54.5 + ] + ], + [ + [ + 36.0 + ] + ], + [ + [ + 54.0 + ] + ], + [ + [ + 49.25 + ] + ], + [ + [ + 61.0 + ] + ], + [ + [ + 61.75 + ] + ], + [ + [ + 71.0 + ] + ], + [ + [ + 78.0 + ] + ], + [ + [ + 71.0 + ] + ], + [ + [ + 90.5 + ] + ] + ] + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6015625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.111328125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.765625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1005859375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.41796875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 21.75 + ] + ] + ], + "outputs": [ + [ + [ + 6.09375 + ] + ], + [ + [ + 1.521856386081038e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.443359375 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 57.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.9375 + ] + ] + ], + "outputs": [ + [ + [ + 15.75 + ] + ], + [ + [ + 28.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.50390625 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.4921875 + ] + ], + [ + [ + 15.4375 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.6875 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4921875 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.875 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.4375 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.6875 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 17.75 + ] + ], + [ + [ + 15.4375 + ] + ], + [ + [ + 8.6875 + ] + ] + ], + "outputs": [ + [ + [ + 5.9375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.22.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 46.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.625 + ] + ] + } + }, + "language_model.model.layers.22.feed_forward.down_proj": { + "inputs": [ + [ + [ + 27.0 + ] + ] + ], + "outputs": [ + [ + [ + 14.875 + ] + ], + [ + [ + 77.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.80859375 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 69.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.25 + ] + ] + ], + "outputs": [ + [ + [ + 70.0 + ] + ], + [ + [ + 79.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2490234375 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.25 + ] + ], + [ + [ + 14.8125 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.875 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.25 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.625 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.8125 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.875 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.1875 + ] + ], + [ + [ + 14.8125 + ] + ], + [ + [ + 11.875 + ] + ] + ], + "outputs": [ + [ + [ + 7.25 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.23.feed_forward.router": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.474609375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 12.875 + ] + ], + [ + [ + 79.0 + ] + ], + [ + [ + 103.5 + ] + ], + [ + [ + 58.5 + ] + ], + [ + [ + 88.0 + ] + ], + [ + [ + 128.0 + ] + ], + [ + [ + 60.75 + ] + ], + [ + [ + 99.5 + ] + ], + [ + [ + 61.5 + ] + ], + [ + [ + 97.0 + ] + ], + [ + [ + 50.5 + ] + ], + [ + [ + 90.0 + ] + ], + [ + [ + 97.0 + ] + ], + [ + [ + 63.0 + ] + ], + [ + [ + 64.0 + ] + ], + [ + [ + 112.5 + ] + ], + [ + [ + 97.0 + ] + ] + ] + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3046875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.349609375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.41796875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.310546875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 1.3359375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 42.25 + ] + ] + ], + "outputs": [ + [ + [ + 19.5 + ] + ], + [ + [ + 1.521856386081038e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.54296875 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.84375 + ] + ] + ], + "outputs": [ + [ + [ + 48.0 + ] + ], + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10986328125 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.40625 + ] + ], + [ + [ + 20.0 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.25 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.40625 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.09375 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.0 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.25 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.5 + ] + ], + [ + [ + 20.0 + ] + ], + [ + [ + 10.25 + ] + ] + ], + "outputs": [ + [ + [ + 4.84375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.24.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5859375 + ] + ] + } + }, + "language_model.model.layers.24.feed_forward.down_proj": { + "inputs": [ + [ + [ + 23.625 + ] + ] + ], + "outputs": [ + [ + [ + 21.75 + ] + ], + [ + [ + 100.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.83203125 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.6875 + ] + ] + ], + "outputs": [ + [ + [ + 53.0 + ] + ], + [ + [ + 69.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.21875 + ] + ], + [ + [ + 17.625 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.5 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.21875 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.09375 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.625 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.5 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.625 + ] + ], + [ + [ + 17.625 + ] + ], + [ + [ + 10.5 + ] + ] + ], + "outputs": [ + [ + [ + 4.6875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.25.feed_forward.router": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.412109375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "outputs": [ + [ + [ + 24.625 + ] + ], + [ + [ + 66.5 + ] + ], + [ + [ + 74.5 + ] + ], + [ + [ + 55.25 + ] + ], + [ + [ + 63.5 + ] + ], + [ + [ + 40.25 + ] + ], + [ + [ + 46.0 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 63.5 + ] + ], + [ + [ + 29.875 + ] + ], + [ + [ + 109.5 + ] + ], + [ + [ + 67.5 + ] + ], + [ + [ + 59.75 + ] + ], + [ + [ + 63.5 + ] + ], + [ + [ + 54.5 + ] + ], + [ + [ + 77.5 + ] + ], + [ + [ + 55.5 + ] + ] + ] + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23828125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.62109375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.390625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 38.25 + ] + ] + ], + "outputs": [ + [ + [ + 9.625 + ] + ], + [ + [ + 1.604866734412731e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.65625 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 63.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.46875 + ] + ] + ], + "outputs": [ + [ + [ + 30.875 + ] + ], + [ + [ + 48.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2109375 + ] + ], + [ + [ + 22.25 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.0 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2109375 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.5 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 22.25 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.0 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.25 + ] + ], + [ + [ + 22.25 + ] + ], + [ + [ + 10.0 + ] + ] + ], + "outputs": [ + [ + [ + 7.46875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.26.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 45.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.365234375 + ] + ] + } + }, + "language_model.model.layers.26.feed_forward.down_proj": { + "inputs": [ + [ + [ + 28.25 + ] + ] + ], + "outputs": [ + [ + [ + 22.125 + ] + ], + [ + [ + 113.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.57421875 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.8125 + ] + ] + ], + "outputs": [ + [ + [ + 60.25 + ] + ], + [ + [ + 71.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.125 + ] + ], + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.625 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.125 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.21875 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 14.625 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.8125 + ] + ], + [ + [ + 17.5 + ] + ], + [ + [ + 14.625 + ] + ] + ], + "outputs": [ + [ + [ + 8.8125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.27.feed_forward.router": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "outputs": [ + [ + [ + 14.25 + ] + ], + [ + [ + 78.0 + ] + ], + [ + [ + 62.25 + ] + ], + [ + [ + 114.0 + ] + ], + [ + [ + 65.5 + ] + ], + [ + [ + 74.0 + ] + ], + [ + [ + 79.5 + ] + ], + [ + [ + 50.25 + ] + ], + [ + [ + 66.5 + ] + ], + [ + [ + 82.0 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 57.0 + ] + ], + [ + [ + 56.25 + ] + ], + [ + [ + 97.0 + ] + ], + [ + [ + 32.5 + ] + ], + [ + [ + 74.5 + ] + ], + [ + [ + 47.25 + ] + ] + ] + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.41015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2490234375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34765625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.384765625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09326171875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.111328125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4140625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "outputs": [ + [ + [ + 48.75 + ] + ], + [ + [ + 1.604866734412731e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.70703125 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 78.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.09375 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 97.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09326171875 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.203125 + ] + ], + [ + [ + 16.625 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.0625 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.203125 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.03125 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.0625 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.75 + ] + ], + [ + [ + 16.625 + ] + ], + [ + [ + 10.0 + ] + ] + ], + "outputs": [ + [ + [ + 4.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.28.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.28.feed_forward.down_proj": { + "inputs": [ + [ + [ + 32.5 + ] + ] + ], + "outputs": [ + [ + [ + 140.0 + ] + ], + [ + [ + 149.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6953125 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 67.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.46875 + ] + ] + ], + "outputs": [ + [ + [ + 78.0 + ] + ], + [ + [ + 84.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.5390625 + ] + ], + [ + [ + 19.5 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.59375 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.5390625 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.25 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.5 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.59375 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 18.75 + ] + ], + [ + [ + 19.5 + ] + ], + [ + [ + 6.59375 + ] + ] + ], + "outputs": [ + [ + [ + 5.46875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.29.feed_forward.router": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.333984375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "outputs": [ + [ + [ + 11.9375 + ] + ], + [ + [ + 74.5 + ] + ], + [ + [ + 55.0 + ] + ], + [ + [ + 93.5 + ] + ], + [ + [ + 52.25 + ] + ], + [ + [ + 95.5 + ] + ], + [ + [ + 62.5 + ] + ], + [ + [ + 55.75 + ] + ], + [ + [ + 80.5 + ] + ], + [ + [ + 63.25 + ] + ], + [ + [ + 74.5 + ] + ], + [ + [ + 39.75 + ] + ], + [ + [ + 17.75 + ] + ], + [ + [ + 45.25 + ] + ], + [ + [ + 69.5 + ] + ], + [ + [ + 70.0 + ] + ], + [ + [ + 57.25 + ] + ] + ] + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2490234375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.240234375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.369140625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 42.75 + ] + ] + ], + "outputs": [ + [ + [ + 18.0 + ] + ], + [ + [ + 1.521856386081038e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.50390625 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.o_proj": { + "inputs": [ + [ + [ + 9.6875 + ] + ] + ], + "outputs": [ + [ + [ + 50.25 + ] + ], + [ + [ + 56.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.203125 + ] + ], + [ + [ + 19.0 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.75 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.203125 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.875 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.0 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.75 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 17.75 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 9.75 + ] + ] + ], + "outputs": [ + [ + [ + 9.6875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.30.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 32.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.380859375 + ] + ] + } + }, + "language_model.model.layers.30.feed_forward.down_proj": { + "inputs": [ + [ + [ + 27.5 + ] + ] + ], + "outputs": [ + [ + [ + 30.25 + ] + ], + [ + [ + 159.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6015625 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.o_proj": { + "inputs": [ + [ + [ + 9.5 + ] + ] + ], + "outputs": [ + [ + [ + 77.0 + ] + ], + [ + [ + 115.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2890625 + ] + ], + [ + [ + 14.125 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.0625 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2890625 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.1875 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.125 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 14.0625 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.875 + ] + ], + [ + [ + 14.125 + ] + ], + [ + [ + 14.0625 + ] + ] + ], + "outputs": [ + [ + [ + 9.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.31.feed_forward.router": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "outputs": [ + [ + [ + 20.0 + ] + ], + [ + [ + 49.25 + ] + ], + [ + [ + 107.0 + ] + ], + [ + [ + 38.0 + ] + ], + [ + [ + 99.5 + ] + ], + [ + [ + 82.0 + ] + ], + [ + [ + 61.5 + ] + ], + [ + [ + 63.75 + ] + ], + [ + [ + 21.25 + ] + ], + [ + [ + 106.5 + ] + ], + [ + [ + 156.0 + ] + ], + [ + [ + 82.5 + ] + ], + [ + [ + 61.0 + ] + ], + [ + [ + 39.0 + ] + ], + [ + [ + 30.75 + ] + ], + [ + [ + 62.75 + ] + ], + [ + [ + 45.0 + ] + ] + ] + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.333984375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.380859375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10498046875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08984375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5234375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 52.75 + ] + ] + ], + "outputs": [ + [ + [ + 25.25 + ] + ], + [ + [ + 1.521856386081038e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.734375 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 53.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.3125 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 109.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8046875 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.140625 + ] + ], + [ + [ + 16.125 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.75 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.140625 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.09375 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.125 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 18.5 + ] + ], + [ + [ + 16.125 + ] + ], + [ + [ + 8.75 + ] + ] + ], + "outputs": [ + [ + [ + 7.3125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.32.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 32.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7734375 + ] + ] + } + }, + "language_model.model.layers.32.feed_forward.down_proj": { + "inputs": [ + [ + [ + 62.75 + ] + ] + ], + "outputs": [ + [ + [ + 29.375 + ] + ], + [ + [ + 75.5 + ] + ] + ], + "params": { + "weight": [ + [ + 1.0078125 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 44.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.375 + ] + ] + ], + "outputs": [ + [ + [ + 94.0 + ] + ], + [ + [ + 132.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.44921875 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.421875 + ] + ], + [ + [ + 16.25 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.5 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.421875 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.03125 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.25 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.5 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 17.5 + ] + ], + [ + [ + 16.25 + ] + ], + [ + [ + 10.5 + ] + ] + ], + "outputs": [ + [ + [ + 6.375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.33.feed_forward.router": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "outputs": [ + [ + [ + 15.6875 + ] + ], + [ + [ + 127.5 + ] + ], + [ + [ + 100.0 + ] + ], + [ + [ + 106.5 + ] + ], + [ + [ + 79.0 + ] + ], + [ + [ + 56.0 + ] + ], + [ + [ + 103.0 + ] + ], + [ + [ + 87.0 + ] + ], + [ + [ + 79.5 + ] + ], + [ + [ + 97.5 + ] + ], + [ + [ + 111.5 + ] + ], + [ + [ + 134.0 + ] + ], + [ + [ + 87.5 + ] + ], + [ + [ + 80.0 + ] + ], + [ + [ + 129.0 + ] + ], + [ + [ + 98.5 + ] + ], + [ + [ + 73.5 + ] + ] + ] + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.431640625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3046875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1025390625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 70.0 + ] + ] + ], + "outputs": [ + [ + [ + 16.125 + ] + ], + [ + [ + 1.8354510353341004e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.490234375 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 54.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.09375 + ] + ] + ], + "outputs": [ + [ + [ + 55.75 + ] + ], + [ + [ + 98.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.5 + ] + ], + [ + [ + 16.75 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.1875 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.5 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.78125 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.75 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.1875 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 18.25 + ] + ], + [ + [ + 16.75 + ] + ], + [ + [ + 7.1875 + ] + ] + ], + "outputs": [ + [ + [ + 6.09375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.34.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.34.feed_forward.down_proj": { + "inputs": [ + [ + [ + 100.0 + ] + ] + ], + "outputs": [ + [ + [ + 36.5 + ] + ], + [ + [ + 162.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6875 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 54.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.78125 + ] + ] + ], + "outputs": [ + [ + [ + 110.5 + ] + ], + [ + [ + 116.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1015625 + ] + ], + [ + [ + 15.3125 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.0 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1015625 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.03125 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.3125 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.0 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.875 + ] + ], + [ + [ + 15.3125 + ] + ], + [ + [ + 9.0 + ] + ] + ], + "outputs": [ + [ + [ + 6.78125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.35.feed_forward.router": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "outputs": [ + [ + [ + 42.5 + ] + ], + [ + [ + 141.0 + ] + ], + [ + [ + 87.0 + ] + ], + [ + [ + 97.0 + ] + ], + [ + [ + 75.0 + ] + ], + [ + [ + 172.0 + ] + ], + [ + [ + 21.75 + ] + ], + [ + [ + 104.0 + ] + ], + [ + [ + 125.5 + ] + ], + [ + [ + 97.5 + ] + ], + [ + [ + 100.0 + ] + ], + [ + [ + 159.0 + ] + ], + [ + [ + 135.0 + ] + ], + [ + [ + 177.0 + ] + ], + [ + [ + 117.5 + ] + ], + [ + [ + 100.0 + ] + ], + [ + [ + 100.0 + ] + ] + ] + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09130859375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.69140625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.359375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23828125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28515625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11181640625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.396484375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 102.0 + ] + ] + ], + "outputs": [ + [ + [ + 10.1875 + ] + ], + [ + [ + 1.521856386081038e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.404296875 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 56.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.875 + ] + ] + ], + "outputs": [ + [ + [ + 36.75 + ] + ], + [ + [ + 57.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.8671875 + ] + ], + [ + [ + 12.6875 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.0 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.8671875 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.65625 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 12.6875 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.0 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 20.75 + ] + ], + [ + [ + 12.6875 + ] + ], + [ + [ + 6.0 + ] + ] + ], + "outputs": [ + [ + [ + 4.875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.36.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 29.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.345703125 + ] + ] + } + }, + "language_model.model.layers.36.feed_forward.down_proj": { + "inputs": [ + [ + [ + 84.0 + ] + ] + ], + "outputs": [ + [ + [ + 43.0 + ] + ], + [ + [ + 94.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.73828125 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.o_proj": { + "inputs": [ + [ + [ + 13.5 + ] + ] + ], + "outputs": [ + [ + [ + 97.5 + ] + ], + [ + [ + 150.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 2.34375 + ] + ], + [ + [ + 15.0 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 17.375 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.34375 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.875 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.0 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 17.375 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 27.75 + ] + ], + [ + [ + 15.0 + ] + ], + [ + [ + 17.375 + ] + ] + ], + "outputs": [ + [ + [ + 13.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.37.feed_forward.router": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "outputs": [ + [ + [ + 44.25 + ] + ], + [ + [ + 100.0 + ] + ], + [ + [ + 115.5 + ] + ], + [ + [ + 127.5 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 129.0 + ] + ], + [ + [ + 66.5 + ] + ], + [ + [ + 152.0 + ] + ], + [ + [ + 131.0 + ] + ], + [ + [ + 189.0 + ] + ], + [ + [ + 143.0 + ] + ], + [ + [ + 121.0 + ] + ], + [ + [ + 95.0 + ] + ], + [ + [ + 140.0 + ] + ], + [ + [ + 151.0 + ] + ], + [ + [ + 166.0 + ] + ], + [ + [ + 194.0 + ] + ] + ] + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.244140625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.73046875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.283203125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09326171875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.361328125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.412109375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 112.5 + ] + ] + ], + "outputs": [ + [ + [ + 19.375 + ] + ], + [ + [ + 1.521856386081038e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4765625 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 47.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.o_proj": { + "inputs": [ + [ + [ + 9.9375 + ] + ] + ], + "outputs": [ + [ + [ + 29.75 + ] + ], + [ + [ + 66.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 2.234375 + ] + ], + [ + [ + 15.625 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 15.0625 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.234375 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.25 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.625 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 15.0625 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 27.0 + ] + ], + [ + [ + 14.5 + ] + ], + [ + [ + 15.0625 + ] + ] + ], + "outputs": [ + [ + [ + 9.9375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.38.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 31.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.435546875 + ] + ] + } + }, + "language_model.model.layers.38.feed_forward.down_proj": { + "inputs": [ + [ + [ + 93.5 + ] + ] + ], + "outputs": [ + [ + [ + 31.125 + ] + ], + [ + [ + 262.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.70703125 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 52.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.o_proj": { + "inputs": [ + [ + [ + 11.875 + ] + ] + ], + "outputs": [ + [ + [ + 126.5 + ] + ], + [ + [ + 154.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.140625 + ] + ], + [ + [ + 16.625 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.0 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.140625 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.75 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 14.0 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.0 + ] + ], + [ + [ + 16.625 + ] + ], + [ + [ + 14.0 + ] + ] + ], + "outputs": [ + [ + [ + 11.875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.39.feed_forward.router": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "outputs": [ + [ + [ + 30.375 + ] + ], + [ + [ + 125.0 + ] + ], + [ + [ + 161.0 + ] + ], + [ + [ + 126.5 + ] + ], + [ + [ + 75.0 + ] + ], + [ + [ + 72.5 + ] + ], + [ + [ + 121.0 + ] + ], + [ + [ + 201.0 + ] + ], + [ + [ + 146.0 + ] + ], + [ + [ + 372.0 + ] + ], + [ + [ + 115.5 + ] + ], + [ + [ + 131.0 + ] + ], + [ + [ + 130.0 + ] + ], + [ + [ + 138.0 + ] + ], + [ + [ + 159.0 + ] + ], + [ + [ + 94.0 + ] + ], + [ + [ + 214.0 + ] + ] + ] + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.400390625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08837890625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.419921875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.484375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 136.0 + ] + ] + ], + "outputs": [ + [ + [ + 90.0 + ] + ], + [ + [ + 1.521856386081038e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.84375 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 46.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.o_proj": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "outputs": [ + [ + [ + 94.0 + ] + ], + [ + [ + 102.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 2.0 + ] + ], + [ + [ + 13.125 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 18.5 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.0 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.9375 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.125 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 24.0 + ] + ], + [ + [ + 13.125 + ] + ], + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 12.8125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.40.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.40.feed_forward.down_proj": { + "inputs": [ + [ + [ + 184.0 + ] + ] + ], + "outputs": [ + [ + [ + 28.5 + ] + ], + [ + [ + 127.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.50390625 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 35.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.o_proj": { + "inputs": [ + [ + [ + 13.9375 + ] + ] + ], + "outputs": [ + [ + [ + 160.0 + ] + ], + [ + [ + 177.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.4921875 + ] + ], + [ + [ + 11.5 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 19.75 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4921875 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.75 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 19.75 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 18.0 + ] + ], + [ + [ + 11.5 + ] + ], + [ + [ + 19.75 + ] + ] + ], + "outputs": [ + [ + [ + 13.9375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.41.feed_forward.router": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "outputs": [ + [ + [ + 42.75 + ] + ], + [ + [ + 111.0 + ] + ], + [ + [ + 62.0 + ] + ], + [ + [ + 144.0 + ] + ], + [ + [ + 147.0 + ] + ], + [ + [ + 151.0 + ] + ], + [ + [ + 139.0 + ] + ], + [ + [ + 247.0 + ] + ], + [ + [ + 120.5 + ] + ], + [ + [ + 150.0 + ] + ], + [ + [ + 164.0 + ] + ], + [ + [ + 172.0 + ] + ], + [ + [ + 151.0 + ] + ], + [ + [ + 137.0 + ] + ], + [ + [ + 165.0 + ] + ], + [ + [ + 152.0 + ] + ], + [ + [ + 185.0 + ] + ] + ] + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09326171875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.369140625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08984375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.365234375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.373046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 189.0 + ] + ] + ], + "outputs": [ + [ + [ + 90.0 + ] + ], + [ + [ + 1.521856386081038e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.83203125 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 39.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.o_proj": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ], + "outputs": [ + [ + [ + 65.0 + ] + ], + [ + [ + 146.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.46875 + ] + ], + [ + [ + 11.1875 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 19.875 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.46875 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.875 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 11.1875 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 19.875 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.75 + ] + ], + [ + [ + 11.1875 + ] + ], + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 16.625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.42.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 43.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.42.feed_forward.down_proj": { + "inputs": [ + [ + [ + 196.0 + ] + ] + ], + "outputs": [ + [ + [ + 70.5 + ] + ], + [ + [ + 218.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.66015625 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 43.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.8125 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 232.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.359375 + ] + ], + [ + [ + 26.625 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.0 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.359375 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.6875 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 26.625 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.0 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.9375 + ] + ], + [ + [ + 26.125 + ] + ], + [ + [ + 11.0 + ] + ] + ], + "outputs": [ + [ + [ + 8.5625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.43.feed_forward.router": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "outputs": [ + [ + [ + 84.0 + ] + ], + [ + [ + 278.0 + ] + ], + [ + [ + 186.0 + ] + ], + [ + [ + 172.0 + ] + ], + [ + [ + 136.0 + ] + ], + [ + [ + 306.0 + ] + ], + [ + [ + 129.0 + ] + ], + [ + [ + 177.0 + ] + ], + [ + [ + 308.0 + ] + ], + [ + [ + 151.0 + ] + ], + [ + [ + 176.0 + ] + ], + [ + [ + 216.0 + ] + ], + [ + [ + 176.0 + ] + ], + [ + [ + 215.0 + ] + ], + [ + [ + 528.0 + ] + ], + [ + [ + 207.0 + ] + ], + [ + [ + 145.0 + ] + ] + ] + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.240234375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.359375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 246.0 + ] + ] + ], + "outputs": [ + [ + [ + 35.5 + ] + ], + [ + [ + 1.521856386081038e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.61328125 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 29.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.o_proj": { + "inputs": [ + [ + [ + 18.375 + ] + ] + ], + "outputs": [ + [ + [ + 81.0 + ] + ], + [ + [ + 170.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.734375 + ] + ], + [ + [ + 19.5 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 19.125 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.734375 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 16.5 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.5 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 19.125 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 24.0 + ] + ], + [ + [ + 19.5 + ] + ], + [ + [ + 19.125 + ] + ] + ], + "outputs": [ + [ + [ + 18.375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.44.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 42.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.44.feed_forward.down_proj": { + "inputs": [ + [ + [ + 190.0 + ] + ] + ], + "outputs": [ + [ + [ + 75.5 + ] + ], + [ + [ + 200.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.54296875 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 28.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.o_proj": { + "inputs": [ + [ + [ + 14.5625 + ] + ] + ], + "outputs": [ + [ + [ + 170.0 + ] + ], + [ + [ + 258.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.41796875 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.734375 + ] + ], + [ + [ + 17.125 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.75 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.734375 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.0625 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 14.75 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 20.375 + ] + ], + [ + [ + 17.125 + ] + ], + [ + [ + 14.75 + ] + ] + ], + "outputs": [ + [ + [ + 14.5625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.45.feed_forward.router": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "outputs": [ + [ + [ + 21.5 + ] + ], + [ + [ + 78.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 74.5 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 38.25 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 105.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 50.25 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 39.75 + ] + ] + ] + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.32421875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.578125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.73828125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.57421875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 356.0 + ] + ] + ], + "outputs": [ + [ + [ + 36.5 + ] + ], + [ + [ + 1.521856386081038e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 27.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.o_proj": { + "inputs": [ + [ + [ + 28.5 + ] + ] + ], + "outputs": [ + [ + [ + 126.0 + ] + ], + [ + [ + 221.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.875 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.9765625 + ] + ], + [ + [ + 23.625 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 28.5 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.9765625 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 26.375 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 23.625 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 28.5 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 24.5 + ] + ], + [ + [ + 23.625 + ] + ], + [ + [ + 28.5 + ] + ] + ], + "outputs": [ + [ + [ + 28.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.46.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 40.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.46.feed_forward.down_proj": { + "inputs": [ + [ + [ + 296.0 + ] + ] + ], + "outputs": [ + [ + [ + 1680.0 + ] + ], + [ + [ + 2400.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5078125 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.o_proj": { + "inputs": [ + [ + [ + 14.9375 + ] + ] + ], + "outputs": [ + [ + [ + 96.5 + ] + ], + [ + [ + 125.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.50390625 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.74609375 + ] + ], + [ + [ + 7.9375 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.9375 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.74609375 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.9375 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 7.9375 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 14.9375 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 8.875 + ] + ], + [ + [ + 7.9375 + ] + ], + [ + [ + 14.9375 + ] + ] + ], + "outputs": [ + [ + [ + 14.9375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.47.feed_forward.router": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "outputs": [ + [ + [ + 404.0 + ] + ], + [ + [ + 81.0 + ] + ], + [ + [ + 214.0 + ] + ], + [ + [ + 108.0 + ] + ], + [ + [ + 468.0 + ] + ], + [ + [ + 520.0 + ] + ], + [ + [ + 97.5 + ] + ], + [ + [ + 125.0 + ] + ], + [ + [ + 241.0 + ] + ], + [ + [ + 199.0 + ] + ], + [ + [ + 458.0 + ] + ], + [ + [ + 120.0 + ] + ], + [ + [ + 216.0 + ] + ], + [ + [ + 288.0 + ] + ], + [ + [ + 66.5 + ] + ], + [ + [ + 154.0 + ] + ], + [ + [ + 178.0 + ] + ] + ] + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10546875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.84765625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.67578125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.474609375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.765625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.59375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.73046875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.66015625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.75390625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.81640625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7109375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.73046875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5546875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.80078125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.66015625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.38671875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 816.0 + ] + ] + ], + "outputs": [ + [ + [ + 832.0 + ] + ], + [ + [ + 1.604866734412731e+21 + ] + ] + ], + "params": { + "weight": [ + [ + 0.625 + ] + ] + } + }, + "language_model.lm_head": { + "inputs": [ + [ + [ + 54.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8203125 + ] + ] + } + } + } +} \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_0_8.npz b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_0_8.npz new file mode 100644 index 000000000000..5c37ec960c9b Binary files /dev/null and b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_0_8.npz differ diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_0_8_mod_list.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_0_8_mod_list.json new file mode 100644 index 000000000000..449fce78b77d --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_0_8_mod_list.json @@ -0,0 +1,1559 @@ +[ + "vision_model.patch_embedding.linear", + "vision_model.model.layers.0.self_attn.qkv_proj", + "vision_model.model.layers.0.self_attn.o_proj", + "vision_model.model.layers.0.mlp.fc1", + "vision_model.model.layers.0.mlp.fc2", + "vision_model.model.layers.1.self_attn.qkv_proj", + "vision_model.model.layers.1.self_attn.o_proj", + "vision_model.model.layers.1.mlp.fc1", + "vision_model.model.layers.1.mlp.fc2", + "vision_model.model.layers.2.self_attn.qkv_proj", + "vision_model.model.layers.2.self_attn.o_proj", + "vision_model.model.layers.2.mlp.fc1", + "vision_model.model.layers.2.mlp.fc2", + "vision_model.model.layers.3.self_attn.qkv_proj", + "vision_model.model.layers.3.self_attn.o_proj", + "vision_model.model.layers.3.mlp.fc1", + "vision_model.model.layers.3.mlp.fc2", + "vision_model.model.layers.4.self_attn.qkv_proj", + "vision_model.model.layers.4.self_attn.o_proj", + "vision_model.model.layers.4.mlp.fc1", + "vision_model.model.layers.4.mlp.fc2", + "vision_model.model.layers.5.self_attn.qkv_proj", + "vision_model.model.layers.5.self_attn.o_proj", + "vision_model.model.layers.5.mlp.fc1", + "vision_model.model.layers.5.mlp.fc2", + "vision_model.model.layers.6.self_attn.qkv_proj", + "vision_model.model.layers.6.self_attn.o_proj", + "vision_model.model.layers.6.mlp.fc1", + "vision_model.model.layers.6.mlp.fc2", + "vision_model.model.layers.7.self_attn.qkv_proj", + "vision_model.model.layers.7.self_attn.o_proj", + "vision_model.model.layers.7.mlp.fc1", + "vision_model.model.layers.7.mlp.fc2", + "vision_model.model.layers.8.self_attn.qkv_proj", + "vision_model.model.layers.8.self_attn.o_proj", + "vision_model.model.layers.8.mlp.fc1", + "vision_model.model.layers.8.mlp.fc2", + "vision_model.model.layers.9.self_attn.qkv_proj", + "vision_model.model.layers.9.self_attn.o_proj", + "vision_model.model.layers.9.mlp.fc1", + "vision_model.model.layers.9.mlp.fc2", + "vision_model.model.layers.10.self_attn.qkv_proj", + "vision_model.model.layers.10.self_attn.o_proj", + "vision_model.model.layers.10.mlp.fc1", + "vision_model.model.layers.10.mlp.fc2", + "vision_model.model.layers.11.self_attn.qkv_proj", + "vision_model.model.layers.11.self_attn.o_proj", + "vision_model.model.layers.11.mlp.fc1", + "vision_model.model.layers.11.mlp.fc2", + "vision_model.model.layers.12.self_attn.qkv_proj", + "vision_model.model.layers.12.self_attn.o_proj", + "vision_model.model.layers.12.mlp.fc1", + "vision_model.model.layers.12.mlp.fc2", + "vision_model.model.layers.13.self_attn.qkv_proj", + "vision_model.model.layers.13.self_attn.o_proj", + "vision_model.model.layers.13.mlp.fc1", + "vision_model.model.layers.13.mlp.fc2", + "vision_model.model.layers.14.self_attn.qkv_proj", + "vision_model.model.layers.14.self_attn.o_proj", + "vision_model.model.layers.14.mlp.fc1", + "vision_model.model.layers.14.mlp.fc2", + "vision_model.model.layers.15.self_attn.qkv_proj", + "vision_model.model.layers.15.self_attn.o_proj", + "vision_model.model.layers.15.mlp.fc1", + "vision_model.model.layers.15.mlp.fc2", + "vision_model.model.layers.16.self_attn.qkv_proj", + "vision_model.model.layers.16.self_attn.o_proj", + "vision_model.model.layers.16.mlp.fc1", + "vision_model.model.layers.16.mlp.fc2", + "vision_model.model.layers.17.self_attn.qkv_proj", + "vision_model.model.layers.17.self_attn.o_proj", + "vision_model.model.layers.17.mlp.fc1", + "vision_model.model.layers.17.mlp.fc2", + "vision_model.model.layers.18.self_attn.qkv_proj", + "vision_model.model.layers.18.self_attn.o_proj", + "vision_model.model.layers.18.mlp.fc1", + "vision_model.model.layers.18.mlp.fc2", + "vision_model.model.layers.19.self_attn.qkv_proj", + "vision_model.model.layers.19.self_attn.o_proj", + "vision_model.model.layers.19.mlp.fc1", + "vision_model.model.layers.19.mlp.fc2", + "vision_model.model.layers.20.self_attn.qkv_proj", + "vision_model.model.layers.20.self_attn.o_proj", + "vision_model.model.layers.20.mlp.fc1", + "vision_model.model.layers.20.mlp.fc2", + "vision_model.model.layers.21.self_attn.qkv_proj", + "vision_model.model.layers.21.self_attn.o_proj", + "vision_model.model.layers.21.mlp.fc1", + "vision_model.model.layers.21.mlp.fc2", + "vision_model.model.layers.22.self_attn.qkv_proj", + "vision_model.model.layers.22.self_attn.o_proj", + "vision_model.model.layers.22.mlp.fc1", + "vision_model.model.layers.22.mlp.fc2", + "vision_model.model.layers.23.self_attn.qkv_proj", + "vision_model.model.layers.23.self_attn.o_proj", + "vision_model.model.layers.23.mlp.fc1", + "vision_model.model.layers.23.mlp.fc2", + "vision_model.model.layers.24.self_attn.qkv_proj", + "vision_model.model.layers.24.self_attn.o_proj", + "vision_model.model.layers.24.mlp.fc1", + "vision_model.model.layers.24.mlp.fc2", + "vision_model.model.layers.25.self_attn.qkv_proj", + "vision_model.model.layers.25.self_attn.o_proj", + "vision_model.model.layers.25.mlp.fc1", + "vision_model.model.layers.25.mlp.fc2", + "vision_model.model.layers.26.self_attn.qkv_proj", + "vision_model.model.layers.26.self_attn.o_proj", + "vision_model.model.layers.26.mlp.fc1", + "vision_model.model.layers.26.mlp.fc2", + "vision_model.model.layers.27.self_attn.qkv_proj", + "vision_model.model.layers.27.self_attn.o_proj", + "vision_model.model.layers.27.mlp.fc1", + "vision_model.model.layers.27.mlp.fc2", + "vision_model.model.layers.28.self_attn.qkv_proj", + "vision_model.model.layers.28.self_attn.o_proj", + "vision_model.model.layers.28.mlp.fc1", + "vision_model.model.layers.28.mlp.fc2", + "vision_model.model.layers.29.self_attn.qkv_proj", + "vision_model.model.layers.29.self_attn.o_proj", + "vision_model.model.layers.29.mlp.fc1", + "vision_model.model.layers.29.mlp.fc2", + "vision_model.model.layers.30.self_attn.qkv_proj", + "vision_model.model.layers.30.self_attn.o_proj", + "vision_model.model.layers.30.mlp.fc1", + "vision_model.model.layers.30.mlp.fc2", + "vision_model.model.layers.31.self_attn.qkv_proj", + "vision_model.model.layers.31.self_attn.o_proj", + "vision_model.model.layers.31.mlp.fc1", + "vision_model.model.layers.31.mlp.fc2", + "vision_model.model.layers.32.self_attn.qkv_proj", + "vision_model.model.layers.32.self_attn.o_proj", + "vision_model.model.layers.32.mlp.fc1", + "vision_model.model.layers.32.mlp.fc2", + "vision_model.model.layers.33.self_attn.qkv_proj", + "vision_model.model.layers.33.self_attn.o_proj", + "vision_model.model.layers.33.mlp.fc1", + "vision_model.model.layers.33.mlp.fc2", + "vision_model.vision_adapter.mlp.fc1", + "vision_model.vision_adapter.mlp.fc2", + "multi_modal_projector.linear_1", + "language_model.model.layers.0.self_attn.qkv_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.0.self_attn.attn.impl.softmax", + "language_model.model.layers.0.self_attn.attn.impl.matmul_av", + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.0.self_attn.attn.impl.k_cache", + "language_model.model.layers.0.self_attn.attn.impl.v_cache", + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.0.feed_forward.gate_up_proj", + "language_model.model.layers.0.feed_forward.down_proj", + "language_model.model.layers.1.self_attn.qkv_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.1.self_attn.attn.impl.softmax", + "language_model.model.layers.1.self_attn.attn.impl.matmul_av", + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.1.self_attn.attn.impl.k_cache", + "language_model.model.layers.1.self_attn.attn.impl.v_cache", + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.1.feed_forward.router", + "language_model.model.layers.1.feed_forward.experts", + "language_model.model.layers.1.feed_forward.experts.moe_op", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.1.feed_forward.shared_expert.down_proj", + "language_model.model.layers.2.self_attn.qkv_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.2.self_attn.attn.impl.softmax", + "language_model.model.layers.2.self_attn.attn.impl.matmul_av", + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.2.self_attn.attn.impl.k_cache", + "language_model.model.layers.2.self_attn.attn.impl.v_cache", + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.2.feed_forward.gate_up_proj", + "language_model.model.layers.2.feed_forward.down_proj", + "language_model.model.layers.3.self_attn.qkv_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.3.self_attn.attn.impl.softmax", + "language_model.model.layers.3.self_attn.attn.impl.matmul_av", + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.3.self_attn.attn.impl.k_cache", + "language_model.model.layers.3.self_attn.attn.impl.v_cache", + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.3.feed_forward.router", + "language_model.model.layers.3.feed_forward.experts", + "language_model.model.layers.3.feed_forward.experts.moe_op", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.3.feed_forward.shared_expert.down_proj", + "language_model.model.layers.4.self_attn.qkv_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.4.self_attn.attn.impl.softmax", + "language_model.model.layers.4.self_attn.attn.impl.matmul_av", + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.4.self_attn.attn.impl.k_cache", + "language_model.model.layers.4.self_attn.attn.impl.v_cache", + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.4.feed_forward.gate_up_proj", + "language_model.model.layers.4.feed_forward.down_proj", + "language_model.model.layers.5.self_attn.qkv_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.5.self_attn.attn.impl.softmax", + "language_model.model.layers.5.self_attn.attn.impl.matmul_av", + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.5.self_attn.attn.impl.k_cache", + "language_model.model.layers.5.self_attn.attn.impl.v_cache", + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.5.feed_forward.router", + "language_model.model.layers.5.feed_forward.experts", + "language_model.model.layers.5.feed_forward.experts.moe_op", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.5.feed_forward.shared_expert.down_proj", + "language_model.model.layers.6.self_attn.qkv_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.6.self_attn.attn.impl.softmax", + "language_model.model.layers.6.self_attn.attn.impl.matmul_av", + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.6.self_attn.attn.impl.k_cache", + "language_model.model.layers.6.self_attn.attn.impl.v_cache", + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.6.feed_forward.gate_up_proj", + "language_model.model.layers.6.feed_forward.down_proj", + "language_model.model.layers.7.self_attn.qkv_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.7.self_attn.attn.impl.softmax", + "language_model.model.layers.7.self_attn.attn.impl.matmul_av", + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.7.self_attn.attn.impl.k_cache", + "language_model.model.layers.7.self_attn.attn.impl.v_cache", + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.7.feed_forward.router", + "language_model.model.layers.7.feed_forward.experts", + "language_model.model.layers.7.feed_forward.experts.moe_op", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.7.feed_forward.shared_expert.down_proj", + "language_model.model.layers.8.self_attn.qkv_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.8.self_attn.attn.impl.softmax", + "language_model.model.layers.8.self_attn.attn.impl.matmul_av", + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.8.self_attn.attn.impl.k_cache", + "language_model.model.layers.8.self_attn.attn.impl.v_cache", + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.8.feed_forward.gate_up_proj", + "language_model.model.layers.8.feed_forward.down_proj", + "language_model.model.layers.9.self_attn.qkv_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.9.self_attn.attn.impl.softmax", + "language_model.model.layers.9.self_attn.attn.impl.matmul_av", + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.9.self_attn.attn.impl.k_cache", + "language_model.model.layers.9.self_attn.attn.impl.v_cache", + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.9.feed_forward.router", + "language_model.model.layers.9.feed_forward.experts", + "language_model.model.layers.9.feed_forward.experts.moe_op", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.9.feed_forward.shared_expert.down_proj", + "language_model.model.layers.10.self_attn.qkv_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.10.self_attn.attn.impl.softmax", + "language_model.model.layers.10.self_attn.attn.impl.matmul_av", + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.10.self_attn.attn.impl.k_cache", + "language_model.model.layers.10.self_attn.attn.impl.v_cache", + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.10.feed_forward.gate_up_proj", + "language_model.model.layers.10.feed_forward.down_proj", + "language_model.model.layers.11.self_attn.qkv_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.11.self_attn.attn.impl.softmax", + "language_model.model.layers.11.self_attn.attn.impl.matmul_av", + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.11.self_attn.attn.impl.k_cache", + "language_model.model.layers.11.self_attn.attn.impl.v_cache", + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.11.feed_forward.router", + "language_model.model.layers.11.feed_forward.experts", + "language_model.model.layers.11.feed_forward.experts.moe_op", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.11.feed_forward.shared_expert.down_proj", + "language_model.model.layers.12.self_attn.qkv_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.12.self_attn.attn.impl.softmax", + "language_model.model.layers.12.self_attn.attn.impl.matmul_av", + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.12.self_attn.attn.impl.k_cache", + "language_model.model.layers.12.self_attn.attn.impl.v_cache", + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.12.feed_forward.gate_up_proj", + "language_model.model.layers.12.feed_forward.down_proj", + "language_model.model.layers.13.self_attn.qkv_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.13.self_attn.attn.impl.softmax", + "language_model.model.layers.13.self_attn.attn.impl.matmul_av", + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.13.self_attn.attn.impl.k_cache", + "language_model.model.layers.13.self_attn.attn.impl.v_cache", + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.13.feed_forward.router", + "language_model.model.layers.13.feed_forward.experts", + "language_model.model.layers.13.feed_forward.experts.moe_op", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.13.feed_forward.shared_expert.down_proj", + "language_model.model.layers.14.self_attn.qkv_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.14.self_attn.attn.impl.softmax", + "language_model.model.layers.14.self_attn.attn.impl.matmul_av", + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.14.self_attn.attn.impl.k_cache", + "language_model.model.layers.14.self_attn.attn.impl.v_cache", + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.14.feed_forward.gate_up_proj", + "language_model.model.layers.14.feed_forward.down_proj", + "language_model.model.layers.15.self_attn.qkv_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.15.self_attn.attn.impl.softmax", + "language_model.model.layers.15.self_attn.attn.impl.matmul_av", + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.15.self_attn.attn.impl.k_cache", + "language_model.model.layers.15.self_attn.attn.impl.v_cache", + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.15.feed_forward.router", + "language_model.model.layers.15.feed_forward.experts", + "language_model.model.layers.15.feed_forward.experts.moe_op", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.15.feed_forward.shared_expert.down_proj", + "language_model.model.layers.16.self_attn.qkv_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.16.self_attn.attn.impl.softmax", + "language_model.model.layers.16.self_attn.attn.impl.matmul_av", + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.16.self_attn.attn.impl.k_cache", + "language_model.model.layers.16.self_attn.attn.impl.v_cache", + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.16.feed_forward.gate_up_proj", + "language_model.model.layers.16.feed_forward.down_proj", + "language_model.model.layers.17.self_attn.qkv_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.17.self_attn.attn.impl.softmax", + "language_model.model.layers.17.self_attn.attn.impl.matmul_av", + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.17.self_attn.attn.impl.k_cache", + "language_model.model.layers.17.self_attn.attn.impl.v_cache", + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.17.feed_forward.router", + "language_model.model.layers.17.feed_forward.experts", + "language_model.model.layers.17.feed_forward.experts.moe_op", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.17.feed_forward.shared_expert.down_proj", + "language_model.model.layers.18.self_attn.qkv_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.18.self_attn.attn.impl.softmax", + "language_model.model.layers.18.self_attn.attn.impl.matmul_av", + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.18.self_attn.attn.impl.k_cache", + "language_model.model.layers.18.self_attn.attn.impl.v_cache", + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.18.feed_forward.gate_up_proj", + "language_model.model.layers.18.feed_forward.down_proj", + "language_model.model.layers.19.self_attn.qkv_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.19.self_attn.attn.impl.softmax", + "language_model.model.layers.19.self_attn.attn.impl.matmul_av", + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.19.self_attn.attn.impl.k_cache", + "language_model.model.layers.19.self_attn.attn.impl.v_cache", + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.19.feed_forward.router", + "language_model.model.layers.19.feed_forward.experts", + "language_model.model.layers.19.feed_forward.experts.moe_op", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.19.feed_forward.shared_expert.down_proj", + "language_model.model.layers.20.self_attn.qkv_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.20.self_attn.attn.impl.softmax", + "language_model.model.layers.20.self_attn.attn.impl.matmul_av", + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.20.self_attn.attn.impl.k_cache", + "language_model.model.layers.20.self_attn.attn.impl.v_cache", + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.20.feed_forward.gate_up_proj", + "language_model.model.layers.20.feed_forward.down_proj", + "language_model.model.layers.21.self_attn.qkv_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.21.self_attn.attn.impl.softmax", + "language_model.model.layers.21.self_attn.attn.impl.matmul_av", + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.21.self_attn.attn.impl.k_cache", + "language_model.model.layers.21.self_attn.attn.impl.v_cache", + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.21.feed_forward.router", + "language_model.model.layers.21.feed_forward.experts", + "language_model.model.layers.21.feed_forward.experts.moe_op", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.21.feed_forward.shared_expert.down_proj", + "language_model.model.layers.22.self_attn.qkv_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.22.self_attn.attn.impl.softmax", + "language_model.model.layers.22.self_attn.attn.impl.matmul_av", + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.22.self_attn.attn.impl.k_cache", + "language_model.model.layers.22.self_attn.attn.impl.v_cache", + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.22.feed_forward.gate_up_proj", + "language_model.model.layers.22.feed_forward.down_proj", + "language_model.model.layers.23.self_attn.qkv_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.23.self_attn.attn.impl.softmax", + "language_model.model.layers.23.self_attn.attn.impl.matmul_av", + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.23.self_attn.attn.impl.k_cache", + "language_model.model.layers.23.self_attn.attn.impl.v_cache", + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.23.feed_forward.router", + "language_model.model.layers.23.feed_forward.experts", + "language_model.model.layers.23.feed_forward.experts.moe_op", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.23.feed_forward.shared_expert.down_proj", + "language_model.model.layers.24.self_attn.qkv_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.24.self_attn.attn.impl.softmax", + "language_model.model.layers.24.self_attn.attn.impl.matmul_av", + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.24.self_attn.attn.impl.k_cache", + "language_model.model.layers.24.self_attn.attn.impl.v_cache", + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.24.feed_forward.gate_up_proj", + "language_model.model.layers.24.feed_forward.down_proj", + "language_model.model.layers.25.self_attn.qkv_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.25.self_attn.attn.impl.softmax", + "language_model.model.layers.25.self_attn.attn.impl.matmul_av", + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.25.self_attn.attn.impl.k_cache", + "language_model.model.layers.25.self_attn.attn.impl.v_cache", + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.25.feed_forward.router", + "language_model.model.layers.25.feed_forward.experts", + "language_model.model.layers.25.feed_forward.experts.moe_op", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.25.feed_forward.shared_expert.down_proj", + "language_model.model.layers.26.self_attn.qkv_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.26.self_attn.attn.impl.softmax", + "language_model.model.layers.26.self_attn.attn.impl.matmul_av", + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.26.self_attn.attn.impl.k_cache", + "language_model.model.layers.26.self_attn.attn.impl.v_cache", + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.26.feed_forward.gate_up_proj", + "language_model.model.layers.26.feed_forward.down_proj", + "language_model.model.layers.27.self_attn.qkv_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.27.self_attn.attn.impl.softmax", + "language_model.model.layers.27.self_attn.attn.impl.matmul_av", + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.27.self_attn.attn.impl.k_cache", + "language_model.model.layers.27.self_attn.attn.impl.v_cache", + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.27.feed_forward.router", + "language_model.model.layers.27.feed_forward.experts", + "language_model.model.layers.27.feed_forward.experts.moe_op", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.27.feed_forward.shared_expert.down_proj", + "language_model.model.layers.28.self_attn.qkv_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.28.self_attn.attn.impl.softmax", + "language_model.model.layers.28.self_attn.attn.impl.matmul_av", + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.28.self_attn.attn.impl.k_cache", + "language_model.model.layers.28.self_attn.attn.impl.v_cache", + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.28.feed_forward.gate_up_proj", + "language_model.model.layers.28.feed_forward.down_proj", + "language_model.model.layers.29.self_attn.qkv_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.29.self_attn.attn.impl.softmax", + "language_model.model.layers.29.self_attn.attn.impl.matmul_av", + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.29.self_attn.attn.impl.k_cache", + "language_model.model.layers.29.self_attn.attn.impl.v_cache", + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.29.feed_forward.router", + "language_model.model.layers.29.feed_forward.experts", + "language_model.model.layers.29.feed_forward.experts.moe_op", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.29.feed_forward.shared_expert.down_proj", + "language_model.model.layers.30.self_attn.qkv_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.30.self_attn.attn.impl.softmax", + "language_model.model.layers.30.self_attn.attn.impl.matmul_av", + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.30.self_attn.attn.impl.k_cache", + "language_model.model.layers.30.self_attn.attn.impl.v_cache", + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.30.feed_forward.gate_up_proj", + "language_model.model.layers.30.feed_forward.down_proj", + "language_model.model.layers.31.self_attn.qkv_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.31.self_attn.attn.impl.softmax", + "language_model.model.layers.31.self_attn.attn.impl.matmul_av", + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.31.self_attn.attn.impl.k_cache", + "language_model.model.layers.31.self_attn.attn.impl.v_cache", + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.31.feed_forward.router", + "language_model.model.layers.31.feed_forward.experts", + "language_model.model.layers.31.feed_forward.experts.moe_op", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.31.feed_forward.shared_expert.down_proj", + "language_model.model.layers.32.self_attn.qkv_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.32.self_attn.attn.impl.softmax", + "language_model.model.layers.32.self_attn.attn.impl.matmul_av", + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.32.self_attn.attn.impl.k_cache", + "language_model.model.layers.32.self_attn.attn.impl.v_cache", + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.32.feed_forward.gate_up_proj", + "language_model.model.layers.32.feed_forward.down_proj", + "language_model.model.layers.33.self_attn.qkv_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.33.self_attn.attn.impl.softmax", + "language_model.model.layers.33.self_attn.attn.impl.matmul_av", + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.33.self_attn.attn.impl.k_cache", + "language_model.model.layers.33.self_attn.attn.impl.v_cache", + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.33.feed_forward.router", + "language_model.model.layers.33.feed_forward.experts", + "language_model.model.layers.33.feed_forward.experts.moe_op", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.33.feed_forward.shared_expert.down_proj", + "language_model.model.layers.34.self_attn.qkv_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.34.self_attn.attn.impl.softmax", + "language_model.model.layers.34.self_attn.attn.impl.matmul_av", + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.34.self_attn.attn.impl.k_cache", + "language_model.model.layers.34.self_attn.attn.impl.v_cache", + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.34.feed_forward.gate_up_proj", + "language_model.model.layers.34.feed_forward.down_proj", + "language_model.model.layers.35.self_attn.qkv_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.35.self_attn.attn.impl.softmax", + "language_model.model.layers.35.self_attn.attn.impl.matmul_av", + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.35.self_attn.attn.impl.k_cache", + "language_model.model.layers.35.self_attn.attn.impl.v_cache", + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.35.feed_forward.router", + "language_model.model.layers.35.feed_forward.experts", + "language_model.model.layers.35.feed_forward.experts.moe_op", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.35.feed_forward.shared_expert.down_proj", + "language_model.model.layers.36.self_attn.qkv_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.36.self_attn.attn.impl.softmax", + "language_model.model.layers.36.self_attn.attn.impl.matmul_av", + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.36.self_attn.attn.impl.k_cache", + "language_model.model.layers.36.self_attn.attn.impl.v_cache", + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.36.feed_forward.gate_up_proj", + "language_model.model.layers.36.feed_forward.down_proj", + "language_model.model.layers.37.self_attn.qkv_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.37.self_attn.attn.impl.softmax", + "language_model.model.layers.37.self_attn.attn.impl.matmul_av", + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.37.self_attn.attn.impl.k_cache", + "language_model.model.layers.37.self_attn.attn.impl.v_cache", + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.37.feed_forward.router", + "language_model.model.layers.37.feed_forward.experts", + "language_model.model.layers.37.feed_forward.experts.moe_op", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.37.feed_forward.shared_expert.down_proj", + "language_model.model.layers.38.self_attn.qkv_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.38.self_attn.attn.impl.softmax", + "language_model.model.layers.38.self_attn.attn.impl.matmul_av", + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.38.self_attn.attn.impl.k_cache", + "language_model.model.layers.38.self_attn.attn.impl.v_cache", + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.38.feed_forward.gate_up_proj", + "language_model.model.layers.38.feed_forward.down_proj", + "language_model.model.layers.39.self_attn.qkv_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.39.self_attn.attn.impl.softmax", + "language_model.model.layers.39.self_attn.attn.impl.matmul_av", + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.39.self_attn.attn.impl.k_cache", + "language_model.model.layers.39.self_attn.attn.impl.v_cache", + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.39.feed_forward.router", + "language_model.model.layers.39.feed_forward.experts", + "language_model.model.layers.39.feed_forward.experts.moe_op", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.39.feed_forward.shared_expert.down_proj", + "language_model.model.layers.40.self_attn.qkv_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.40.self_attn.attn.impl.softmax", + "language_model.model.layers.40.self_attn.attn.impl.matmul_av", + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.40.self_attn.attn.impl.k_cache", + "language_model.model.layers.40.self_attn.attn.impl.v_cache", + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.40.feed_forward.gate_up_proj", + "language_model.model.layers.40.feed_forward.down_proj", + "language_model.model.layers.41.self_attn.qkv_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.41.self_attn.attn.impl.softmax", + "language_model.model.layers.41.self_attn.attn.impl.matmul_av", + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.41.self_attn.attn.impl.k_cache", + "language_model.model.layers.41.self_attn.attn.impl.v_cache", + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.41.feed_forward.router", + "language_model.model.layers.41.feed_forward.experts", + "language_model.model.layers.41.feed_forward.experts.moe_op", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.41.feed_forward.shared_expert.down_proj", + "language_model.model.layers.42.self_attn.qkv_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.42.self_attn.attn.impl.softmax", + "language_model.model.layers.42.self_attn.attn.impl.matmul_av", + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.42.self_attn.attn.impl.k_cache", + "language_model.model.layers.42.self_attn.attn.impl.v_cache", + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.42.feed_forward.gate_up_proj", + "language_model.model.layers.42.feed_forward.down_proj", + "language_model.model.layers.43.self_attn.qkv_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.43.self_attn.attn.impl.softmax", + "language_model.model.layers.43.self_attn.attn.impl.matmul_av", + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.43.self_attn.attn.impl.k_cache", + "language_model.model.layers.43.self_attn.attn.impl.v_cache", + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.43.feed_forward.router", + "language_model.model.layers.43.feed_forward.experts", + "language_model.model.layers.43.feed_forward.experts.moe_op", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.43.feed_forward.shared_expert.down_proj", + "language_model.model.layers.44.self_attn.qkv_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.44.self_attn.attn.impl.softmax", + "language_model.model.layers.44.self_attn.attn.impl.matmul_av", + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.44.self_attn.attn.impl.k_cache", + "language_model.model.layers.44.self_attn.attn.impl.v_cache", + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.44.feed_forward.gate_up_proj", + "language_model.model.layers.44.feed_forward.down_proj", + "language_model.model.layers.45.self_attn.qkv_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.45.self_attn.attn.impl.softmax", + "language_model.model.layers.45.self_attn.attn.impl.matmul_av", + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.45.self_attn.attn.impl.k_cache", + "language_model.model.layers.45.self_attn.attn.impl.v_cache", + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.45.feed_forward.router", + "language_model.model.layers.45.feed_forward.experts", + "language_model.model.layers.45.feed_forward.experts.moe_op", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.45.feed_forward.shared_expert.down_proj", + "language_model.model.layers.46.self_attn.qkv_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.46.self_attn.attn.impl.softmax", + "language_model.model.layers.46.self_attn.attn.impl.matmul_av", + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.46.self_attn.attn.impl.k_cache", + "language_model.model.layers.46.self_attn.attn.impl.v_cache", + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.46.feed_forward.gate_up_proj", + "language_model.model.layers.46.feed_forward.down_proj", + "language_model.model.layers.47.self_attn.qkv_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.47.self_attn.attn.impl.softmax", + "language_model.model.layers.47.self_attn.attn.impl.matmul_av", + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.47.self_attn.attn.impl.k_cache", + "language_model.model.layers.47.self_attn.attn.impl.v_cache", + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.47.feed_forward.router", + "language_model.model.layers.47.feed_forward.experts", + "language_model.model.layers.47.feed_forward.experts.moe_op", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.47.feed_forward.shared_expert.down_proj", + "language_model.lm_head" +] \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_1_8.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_1_8.json new file mode 100644 index 000000000000..0c653c98411d --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_1_8.json @@ -0,0 +1,24263 @@ +{ + "GlobalRank": null, + "LocalRank": 1, + "Mode": "DynamicRange", + "Nodes": { + "language_model.model.layers.0.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 11.1875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.322265625 + ] + ] + } + }, + "language_model.model.layers.0.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.34765625 + ] + ] + ], + "outputs": [ + [ + [ + 0.83203125 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.326171875 + ] + ], + [ + [ + 4.1875 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.51171875 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.326171875 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.314453125 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 4.1875 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.51171875 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 4.0625 + ] + ], + [ + [ + 4.1875 + ] + ], + [ + [ + 0.51171875 + ] + ] + ], + "outputs": [ + [ + [ + 0.34765625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.0.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 2.546875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.0.feed_forward.down_proj": { + "inputs": [ + [ + [ + 3.65625 + ] + ] + ], + "outputs": [ + [ + [ + 26.75 + ] + ], + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.75 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 10.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.73046875 + ] + ] + ], + "outputs": [ + [ + [ + 0.67578125 + ] + ], + [ + [ + 1.3984375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.65625 + ] + ], + [ + [ + 10.375 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.84765625 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.65625 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.6875 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 10.375 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.84765625 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 8.0 + ] + ], + [ + [ + 10.375 + ] + ], + [ + [ + 0.82421875 + ] + ] + ], + "outputs": [ + [ + [ + 0.73046875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.1.feed_forward.router": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "outputs": [ + [ + [ + 1.765625 + ] + ], + [ + [ + 27.5 + ] + ], + [ + [ + 20.75 + ] + ], + [ + [ + 19.375 + ] + ], + [ + [ + 15.9375 + ] + ], + [ + [ + 18.375 + ] + ], + [ + [ + 23.625 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 16.125 + ] + ], + [ + [ + 13.1875 + ] + ], + [ + [ + 6.53125 + ] + ], + [ + [ + 15.75 + ] + ], + [ + [ + 17.125 + ] + ], + [ + [ + 17.375 + ] + ], + [ + [ + 14.8125 + ] + ], + [ + [ + 23.125 + ] + ], + [ + [ + 31.5 + ] + ] + ] + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.083984375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.111328125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.07763671875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.244140625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 6.0 + ] + ] + ], + "outputs": [ + [ + [ + 0.88671875 + ] + ], + [ + [ + 8.174752174077233e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.404296875 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 9.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.96484375 + ] + ] + ], + "outputs": [ + [ + [ + 0.7734375 + ] + ], + [ + [ + 1.4921875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.58203125 + ] + ], + [ + [ + 8.9375 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1796875 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.58203125 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.91796875 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 8.9375 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.1796875 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 6.96875 + ] + ], + [ + [ + 8.9375 + ] + ], + [ + [ + 1.1796875 + ] + ] + ], + "outputs": [ + [ + [ + 0.96484375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.2.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 6.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.2.feed_forward.down_proj": { + "inputs": [ + [ + [ + 4.71875 + ] + ] + ], + "outputs": [ + [ + [ + 4.75 + ] + ], + [ + [ + 25.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.54296875 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 7.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.35546875 + ] + ] + ], + "outputs": [ + [ + [ + 0.77734375 + ] + ], + [ + [ + 2.40625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.41796875 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.05322265625 + ] + ], + [ + [ + 2.796875 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.263671875 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.05322265625 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.263671875 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 2.796875 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.3515625 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 1.625 + ] + ], + [ + [ + 2.796875 + ] + ], + [ + [ + 0.36328125 + ] + ] + ], + "outputs": [ + [ + [ + 0.35546875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.3.feed_forward.router": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.52734375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "outputs": [ + [ + [ + 1.96875 + ] + ], + [ + [ + 26.5 + ] + ], + [ + [ + 17.875 + ] + ], + [ + [ + 22.125 + ] + ], + [ + [ + 22.25 + ] + ], + [ + [ + 12.0 + ] + ], + [ + [ + 18.5 + ] + ], + [ + [ + 18.375 + ] + ], + [ + [ + 20.125 + ] + ], + [ + [ + 16.5 + ] + ], + [ + [ + 6.125 + ] + ], + [ + [ + 16.25 + ] + ], + [ + [ + 18.125 + ] + ], + [ + [ + 11.25 + ] + ], + [ + [ + 20.375 + ] + ], + [ + [ + 14.1875 + ] + ], + [ + [ + 10.5 + ] + ] + ] + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1083984375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1044921875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1005859375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08447265625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08447265625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 6.3125 + ] + ] + ], + "outputs": [ + [ + [ + 0.89453125 + ] + ], + [ + [ + 2.9110093107689658e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5078125 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 14.3125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.5 + ] + ] + ], + "outputs": [ + [ + [ + 1.46875 + ] + ], + [ + [ + 2.328125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.55859375 + ] + ], + [ + [ + 10.4375 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.5 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.55859375 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.171875 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 10.4375 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.5 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 6.3125 + ] + ], + [ + [ + 10.25 + ] + ], + [ + [ + 3.5 + ] + ] + ], + "outputs": [ + [ + [ + 2.453125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.4.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 6.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.4.feed_forward.down_proj": { + "inputs": [ + [ + [ + 7.5625 + ] + ] + ], + "outputs": [ + [ + [ + 2.796875 + ] + ], + [ + [ + 5.1875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.66015625 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 18.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.875 + ] + ] + ], + "outputs": [ + [ + [ + 1.4921875 + ] + ], + [ + [ + 2.40625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.64453125 + ] + ], + [ + [ + 15.4375 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.46875 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.64453125 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.40625 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.4375 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.46875 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.6875 + ] + ], + [ + [ + 15.4375 + ] + ], + [ + [ + 3.46875 + ] + ] + ], + "outputs": [ + [ + [ + 2.875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.5.feed_forward.router": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 1520.0 + ] + ], + [ + [ + 27.0 + ] + ], + [ + [ + 33.5 + ] + ], + [ + [ + 23.875 + ] + ], + [ + [ + 25.875 + ] + ], + [ + [ + 20.125 + ] + ], + [ + [ + 27.875 + ] + ], + [ + [ + 35.25 + ] + ], + [ + [ + 31.5 + ] + ], + [ + [ + 21.5 + ] + ], + [ + [ + 21.375 + ] + ], + [ + [ + 27.375 + ] + ], + [ + [ + 216.0 + ] + ], + [ + [ + 37.75 + ] + ], + [ + [ + 35.75 + ] + ], + [ + [ + 30.75 + ] + ], + [ + [ + 11.6875 + ] + ] + ] + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.431640625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11474609375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10498046875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.78515625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09130859375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 9.25 + ] + ] + ], + "outputs": [ + [ + [ + 0.9296875 + ] + ], + [ + [ + 8.174752174077233e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 14.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.4453125 + ] + ] + ], + "outputs": [ + [ + [ + 0.94140625 + ] + ], + [ + [ + 1.96875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.349609375 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.91015625 + ] + ], + [ + [ + 18.375 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.34375 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.91015625 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.5703125 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.375 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.34375 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.0625 + ] + ], + [ + [ + 18.375 + ] + ], + [ + [ + 2.1875 + ] + ] + ], + "outputs": [ + [ + [ + 1.4453125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.6.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 7.71875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.6.feed_forward.down_proj": { + "inputs": [ + [ + [ + 8.125 + ] + ] + ], + "outputs": [ + [ + [ + 2.25 + ] + ], + [ + [ + 11.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 7.59375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.91796875 + ] + ] + ], + "outputs": [ + [ + [ + 2.1875 + ] + ], + [ + [ + 2.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.392578125 + ] + ], + [ + [ + 4.65625 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.390625 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.392578125 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.60546875 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 4.65625 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.390625 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 4.6875 + ] + ], + [ + [ + 4.4375 + ] + ], + [ + [ + 1.390625 + ] + ] + ], + "outputs": [ + [ + [ + 0.91796875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.7.feed_forward.router": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "outputs": [ + [ + [ + 3.328125 + ] + ], + [ + [ + 26.375 + ] + ], + [ + [ + 24.625 + ] + ], + [ + [ + 38.0 + ] + ], + [ + [ + 25.625 + ] + ], + [ + [ + 27.875 + ] + ], + [ + [ + 34.0 + ] + ], + [ + [ + 43.5 + ] + ], + [ + [ + 31.0 + ] + ], + [ + [ + 44.75 + ] + ], + [ + [ + 30.125 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 9.375 + ] + ], + [ + [ + 27.125 + ] + ], + [ + [ + 14.5 + ] + ], + [ + [ + 37.0 + ] + ], + [ + [ + 34.5 + ] + ] + ] + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10986328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11279296875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11376953125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08642578125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 8.5 + ] + ] + ], + "outputs": [ + [ + [ + 0.51953125 + ] + ], + [ + [ + 1.2428281760588963e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.3046875 + ] + ] + ], + "outputs": [ + [ + [ + 2.265625 + ] + ], + [ + [ + 3.390625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.85546875 + ] + ], + [ + [ + 13.0625 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.1875 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.85546875 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.140625 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.0625 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.1875 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.5 + ] + ], + [ + [ + 13.0625 + ] + ], + [ + [ + 3.1875 + ] + ] + ], + "outputs": [ + [ + [ + 1.3046875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.8.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 8.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.8.feed_forward.down_proj": { + "inputs": [ + [ + [ + 10.6875 + ] + ] + ], + "outputs": [ + [ + [ + 2.84375 + ] + ], + [ + [ + 9.8125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.50390625 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 16.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.0 + ] + ] + ], + "outputs": [ + [ + [ + 1.609375 + ] + ], + [ + [ + 3.640625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.388671875 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.91796875 + ] + ], + [ + [ + 18.0 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.875 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.91796875 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.8359375 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.0 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.875 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.0 + ] + ], + [ + [ + 18.0 + ] + ], + [ + [ + 2.875 + ] + ] + ], + "outputs": [ + [ + [ + 2.0 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.9.feed_forward.router": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "outputs": [ + [ + [ + 7.9375 + ] + ], + [ + [ + 40.5 + ] + ], + [ + [ + 40.75 + ] + ], + [ + [ + 60.25 + ] + ], + [ + [ + 48.0 + ] + ], + [ + [ + 42.0 + ] + ], + [ + [ + 113.5 + ] + ], + [ + [ + 15.1875 + ] + ], + [ + [ + 24.875 + ] + ], + [ + [ + 33.25 + ] + ], + [ + [ + 39.25 + ] + ], + [ + [ + 42.75 + ] + ], + [ + [ + 67.0 + ] + ], + [ + [ + 43.75 + ] + ], + [ + [ + 61.75 + ] + ], + [ + [ + 31.625 + ] + ], + [ + [ + 35.25 + ] + ] + ] + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11572265625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0859375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10888671875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10546875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08642578125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 13.25 + ] + ] + ], + "outputs": [ + [ + [ + 3.296875 + ] + ], + [ + [ + 8.2079828739718555e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6015625 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 17.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.921875 + ] + ] + ], + "outputs": [ + [ + [ + 1.984375 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.60546875 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.8984375 + ] + ], + [ + [ + 9.375 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.875 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.8984375 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.03125 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 9.375 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.875 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 21.75 + ] + ], + [ + [ + 9.375 + ] + ], + [ + [ + 4.875 + ] + ] + ], + "outputs": [ + [ + [ + 3.921875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.10.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 9.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.10.feed_forward.down_proj": { + "inputs": [ + [ + [ + 17.375 + ] + ] + ], + "outputs": [ + [ + [ + 3.40625 + ] + ], + [ + [ + 12.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34765625 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 15.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.125 + ] + ] + ], + "outputs": [ + [ + [ + 8.625 + ] + ], + [ + [ + 8.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.62109375 + ] + ], + [ + [ + 9.6875 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.5 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.62109375 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.09375 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 9.6875 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.5 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.5625 + ] + ], + [ + [ + 9.6875 + ] + ], + [ + [ + 5.5 + ] + ] + ], + "outputs": [ + [ + [ + 4.09375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.11.feed_forward.router": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "outputs": [ + [ + [ + 4.9375 + ] + ], + [ + [ + 55.0 + ] + ], + [ + [ + 55.25 + ] + ], + [ + [ + 53.0 + ] + ], + [ + [ + 45.25 + ] + ], + [ + [ + 42.5 + ] + ], + [ + [ + 62.5 + ] + ], + [ + [ + 41.0 + ] + ], + [ + [ + 38.75 + ] + ], + [ + [ + 45.5 + ] + ], + [ + [ + 51.5 + ] + ], + [ + [ + 91.0 + ] + ], + [ + [ + 24.625 + ] + ], + [ + [ + 38.0 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 47.5 + ] + ], + [ + [ + 21.625 + ] + ] + ] + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11767578125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23828125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09326171875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09130859375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08642578125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11181640625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 19.0 + ] + ] + ], + "outputs": [ + [ + [ + 3.265625 + ] + ], + [ + [ + 8.2079828739718555e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.65234375 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 20.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.765625 + ] + ] + ], + "outputs": [ + [ + [ + 1.796875 + ] + ], + [ + [ + 3.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0078125 + ] + ], + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.5 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0078125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.03125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.5 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.3125 + ] + ], + [ + [ + 18.625 + ] + ], + [ + [ + 3.5 + ] + ] + ], + "outputs": [ + [ + [ + 2.765625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.12.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 10.6875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.12.feed_forward.down_proj": { + "inputs": [ + [ + [ + 15.375 + ] + ] + ], + "outputs": [ + [ + [ + 9.25 + ] + ], + [ + [ + 20.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8671875 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.46875 + ] + ] + ], + "outputs": [ + [ + [ + 2.015625 + ] + ], + [ + [ + 6.34375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.97265625 + ] + ], + [ + [ + 17.75 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.765625 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.97265625 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.03125 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.75 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.765625 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.1875 + ] + ], + [ + [ + 17.625 + ] + ], + [ + [ + 2.765625 + ] + ] + ], + "outputs": [ + [ + [ + 2.46875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.13.feed_forward.router": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "outputs": [ + [ + [ + 10.0625 + ] + ], + [ + [ + 58.5 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 66.5 + ] + ], + [ + [ + 57.25 + ] + ], + [ + [ + 66.5 + ] + ], + [ + [ + 89.0 + ] + ], + [ + [ + 30.875 + ] + ], + [ + [ + 61.25 + ] + ], + [ + [ + 103.5 + ] + ], + [ + [ + 48.5 + ] + ], + [ + [ + 68.0 + ] + ], + [ + [ + 67.0 + ] + ], + [ + [ + 101.5 + ] + ], + [ + [ + 57.0 + ] + ], + [ + [ + 27.125 + ] + ], + [ + [ + 45.5 + ] + ] + ] + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1083984375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.478515625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3828125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4921875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 12.6875 + ] + ] + ], + "outputs": [ + [ + [ + 1.53125 + ] + ], + [ + [ + 2.2331030329186587e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 20.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.875 + ] + ] + ], + "outputs": [ + [ + [ + 3.546875 + ] + ], + [ + [ + 5.71875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.6953125 + ] + ], + [ + [ + 8.1875 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.125 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.6953125 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.921875 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 8.1875 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.125 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 8.0625 + ] + ], + [ + [ + 8.1875 + ] + ], + [ + [ + 6.125 + ] + ] + ], + "outputs": [ + [ + [ + 5.875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.14.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 12.0625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.43359375 + ] + ] + } + }, + "language_model.model.layers.14.feed_forward.down_proj": { + "inputs": [ + [ + [ + 16.75 + ] + ] + ], + "outputs": [ + [ + [ + 5.375 + ] + ], + [ + [ + 20.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.42578125 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 23.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.75 + ] + ] + ], + "outputs": [ + [ + [ + 20.0 + ] + ], + [ + [ + 21.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.89453125 + ] + ], + [ + [ + 11.0625 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.28125 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.89453125 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.78125 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 11.0625 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.28125 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.3125 + ] + ], + [ + [ + 11.0625 + ] + ], + [ + [ + 5.28125 + ] + ] + ], + "outputs": [ + [ + [ + 4.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.15.feed_forward.router": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 6.9375 + ] + ], + [ + [ + 60.25 + ] + ], + [ + [ + 68.0 + ] + ], + [ + [ + 113.0 + ] + ], + [ + [ + 73.0 + ] + ], + [ + [ + 42.0 + ] + ], + [ + [ + 53.5 + ] + ], + [ + [ + 38.0 + ] + ], + [ + [ + 57.0 + ] + ], + [ + [ + 28.25 + ] + ], + [ + [ + 61.5 + ] + ], + [ + [ + 52.0 + ] + ], + [ + [ + 57.0 + ] + ], + [ + [ + 67.0 + ] + ], + [ + [ + 89.5 + ] + ], + [ + [ + 56.25 + ] + ], + [ + [ + 38.0 + ] + ] + ] + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0869140625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.103515625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 27.875 + ] + ] + ], + "outputs": [ + [ + [ + 11.6875 + ] + ], + [ + [ + 2.2331030329186587e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4453125 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 23.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.953125 + ] + ] + ], + "outputs": [ + [ + [ + 2.4375 + ] + ], + [ + [ + 6.3125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.95703125 + ] + ], + [ + [ + 19.375 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.625 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.95703125 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.125 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.375 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.625 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.0625 + ] + ], + [ + [ + 19.375 + ] + ], + [ + [ + 4.625 + ] + ] + ], + "outputs": [ + [ + [ + 3.953125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.16.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 14.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.16.feed_forward.down_proj": { + "inputs": [ + [ + [ + 36.75 + ] + ] + ], + "outputs": [ + [ + [ + 6.65625 + ] + ], + [ + [ + 27.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6796875 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.75 + ] + ] + ], + "outputs": [ + [ + [ + 19.5 + ] + ], + [ + [ + 27.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1015625 + ] + ], + [ + [ + 13.8125 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.90625 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1015625 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.640625 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.8125 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.90625 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.875 + ] + ], + [ + [ + 13.8125 + ] + ], + [ + [ + 4.90625 + ] + ] + ], + "outputs": [ + [ + [ + 4.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.17.feed_forward.router": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 6.8125 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 70.0 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 53.75 + ] + ], + [ + [ + 75.0 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 75.0 + ] + ], + [ + [ + 60.5 + ] + ], + [ + [ + 37.75 + ] + ], + [ + [ + 39.0 + ] + ], + [ + [ + 70.5 + ] + ], + [ + [ + 88.0 + ] + ], + [ + [ + 81.0 + ] + ], + [ + [ + 41.25 + ] + ], + [ + [ + 51.5 + ] + ], + [ + [ + 65.5 + ] + ] + ] + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2353515625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33984375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33203125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.35546875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2353515625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 24.875 + ] + ] + ], + "outputs": [ + [ + [ + 9.5 + ] + ], + [ + [ + 8.2079828739718555e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.44140625 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.59375 + ] + ] + ], + "outputs": [ + [ + [ + 6.625 + ] + ], + [ + [ + 12.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.046875 + ] + ], + [ + [ + 19.25 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.84375 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.046875 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.6875 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.25 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.84375 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.5625 + ] + ], + [ + [ + 19.25 + ] + ], + [ + [ + 6.84375 + ] + ] + ], + "outputs": [ + [ + [ + 5.59375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.18.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 20.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4140625 + ] + ] + } + }, + "language_model.model.layers.18.feed_forward.down_proj": { + "inputs": [ + [ + [ + 16.75 + ] + ] + ], + "outputs": [ + [ + [ + 8.1875 + ] + ], + [ + [ + 354.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.75 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 44.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.34375 + ] + ] + ], + "outputs": [ + [ + [ + 31.5 + ] + ], + [ + [ + 37.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.984375 + ] + ], + [ + [ + 14.1875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.3125 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.984375 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.3125 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.1875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.3125 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.0625 + ] + ], + [ + [ + 13.75 + ] + ], + [ + [ + 6.3125 + ] + ] + ], + "outputs": [ + [ + [ + 5.34375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.19.feed_forward.router": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "outputs": [ + [ + [ + 8.625 + ] + ], + [ + [ + 60.25 + ] + ], + [ + [ + 67.0 + ] + ], + [ + [ + 48.25 + ] + ], + [ + [ + 71.0 + ] + ], + [ + [ + 20.5 + ] + ], + [ + [ + 33.25 + ] + ], + [ + [ + 57.75 + ] + ], + [ + [ + 40.75 + ] + ], + [ + [ + 72.0 + ] + ], + [ + [ + 73.5 + ] + ], + [ + [ + 54.5 + ] + ], + [ + [ + 59.5 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 54.75 + ] + ], + [ + [ + 64.5 + ] + ], + [ + [ + 88.5 + ] + ] + ] + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33984375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33984375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.400390625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.408203125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.345703125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10986328125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33203125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 22.125 + ] + ] + ], + "outputs": [ + [ + [ + 4.0625 + ] + ], + [ + [ + 2.9110093107689658e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3984375 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 53.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.84375 + ] + ] + ], + "outputs": [ + [ + [ + 12.8125 + ] + ], + [ + [ + 22.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.421875 + ] + ], + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.25 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.421875 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.125 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.25 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.0625 + ] + ], + [ + [ + 18.625 + ] + ], + [ + [ + 7.09375 + ] + ] + ], + "outputs": [ + [ + [ + 6.84375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.20.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 42.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.20.feed_forward.down_proj": { + "inputs": [ + [ + [ + 19.625 + ] + ] + ], + "outputs": [ + [ + [ + 10.5 + ] + ], + [ + [ + 41.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.91015625 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 58.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.0625 + ] + ] + ], + "outputs": [ + [ + [ + 26.0 + ] + ], + [ + [ + 35.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0625 + ] + ], + [ + [ + 20.75 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.78125 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0625 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.953125 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.75 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.78125 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.75 + ] + ], + [ + [ + 20.75 + ] + ], + [ + [ + 6.78125 + ] + ] + ], + "outputs": [ + [ + [ + 5.0625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.21.feed_forward.router": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.45703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "outputs": [ + [ + [ + 14.4375 + ] + ], + [ + [ + 126.5 + ] + ], + [ + [ + 25.5 + ] + ], + [ + [ + 63.75 + ] + ], + [ + [ + 59.5 + ] + ], + [ + [ + 81.5 + ] + ], + [ + [ + 87.5 + ] + ], + [ + [ + 81.0 + ] + ], + [ + [ + 59.75 + ] + ], + [ + [ + 47.25 + ] + ], + [ + [ + 100.5 + ] + ], + [ + [ + 31.0 + ] + ], + [ + [ + 53.5 + ] + ], + [ + [ + 58.5 + ] + ], + [ + [ + 22.25 + ] + ], + [ + [ + 137.0 + ] + ], + [ + [ + 65.5 + ] + ] + ] + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.470703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.341796875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.416015625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "outputs": [ + [ + [ + 5.75 + ] + ], + [ + [ + 2.9110093107689658e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.330078125 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 57.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.15625 + ] + ] + ], + "outputs": [ + [ + [ + 15.75 + ] + ], + [ + [ + 28.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.15625 + ] + ], + [ + [ + 20.75 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.65625 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.15625 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.65625 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.75 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.65625 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.6875 + ] + ], + [ + [ + 20.75 + ] + ], + [ + [ + 4.5625 + ] + ] + ], + "outputs": [ + [ + [ + 4.15625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.22.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 46.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.361328125 + ] + ] + } + }, + "language_model.model.layers.22.feed_forward.down_proj": { + "inputs": [ + [ + [ + 19.875 + ] + ] + ], + "outputs": [ + [ + [ + 14.875 + ] + ], + [ + [ + 77.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.78515625 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 69.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.875 + ] + ] + ], + "outputs": [ + [ + [ + 70.0 + ] + ], + [ + [ + 79.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.310546875 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.21875 + ] + ], + [ + [ + 14.5625 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.3125 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.21875 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.0 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.5625 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.3125 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.5625 + ] + ], + [ + [ + 14.5625 + ] + ], + [ + [ + 13.3125 + ] + ] + ], + "outputs": [ + [ + [ + 7.34375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.23.feed_forward.router": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.474609375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 13.4375 + ] + ], + [ + [ + 54.5 + ] + ], + [ + [ + 37.0 + ] + ], + [ + [ + 33.25 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 31.375 + ] + ], + [ + [ + 101.0 + ] + ], + [ + [ + 43.5 + ] + ], + [ + [ + 156.0 + ] + ], + [ + [ + 53.75 + ] + ], + [ + [ + 80.5 + ] + ], + [ + [ + 79.0 + ] + ], + [ + [ + 110.5 + ] + ], + [ + [ + 98.0 + ] + ], + [ + [ + 64.5 + ] + ], + [ + [ + 163.0 + ] + ], + [ + [ + 77.5 + ] + ] + ] + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.283203125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3203125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10546875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.345703125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 67.0 + ] + ] + ], + "outputs": [ + [ + [ + 20.875 + ] + ], + [ + [ + 8.174752174077233e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.431640625 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.5 + ] + ] + ], + "outputs": [ + [ + [ + 48.0 + ] + ], + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.265625 + ] + ], + [ + [ + 15.3125 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.625 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.265625 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.625 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.3125 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.625 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.875 + ] + ], + [ + [ + 15.3125 + ] + ], + [ + [ + 7.625 + ] + ] + ], + "outputs": [ + [ + [ + 5.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.24.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28515625 + ] + ] + } + }, + "language_model.model.layers.24.feed_forward.down_proj": { + "inputs": [ + [ + [ + 160.0 + ] + ] + ], + "outputs": [ + [ + [ + 21.75 + ] + ], + [ + [ + 100.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4921875 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.o_proj": { + "inputs": [ + [ + [ + 10.3125 + ] + ] + ], + "outputs": [ + [ + [ + 53.0 + ] + ], + [ + [ + 69.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.65234375 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1796875 + ] + ], + [ + [ + 19.0 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.3125 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1796875 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.59375 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.0 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.3125 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.25 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 11.3125 + ] + ] + ], + "outputs": [ + [ + [ + 10.3125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.25.feed_forward.router": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.412109375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "outputs": [ + [ + [ + 10.0625 + ] + ], + [ + [ + 63.5 + ] + ], + [ + [ + 66.5 + ] + ], + [ + [ + 52.25 + ] + ], + [ + [ + 64.5 + ] + ], + [ + [ + 63.0 + ] + ], + [ + [ + 61.75 + ] + ], + [ + [ + 182.0 + ] + ], + [ + [ + 71.5 + ] + ], + [ + [ + 49.25 + ] + ], + [ + [ + 29.5 + ] + ], + [ + [ + 54.5 + ] + ], + [ + [ + 80.0 + ] + ], + [ + [ + 75.5 + ] + ], + [ + [ + 78.5 + ] + ], + [ + [ + 107.0 + ] + ], + [ + [ + 69.0 + ] + ] + ] + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.32421875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11474609375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11376953125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.54296875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 27.125 + ] + ] + ], + "outputs": [ + [ + [ + 7.09375 + ] + ], + [ + [ + 8.174752174077233e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.57421875 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 63.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.53125 + ] + ] + ], + "outputs": [ + [ + [ + 30.875 + ] + ], + [ + [ + 48.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.369140625 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.3046875 + ] + ], + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.5625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.3046875 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.5 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.5625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.3125 + ] + ], + [ + [ + 18.625 + ] + ], + [ + [ + 8.5625 + ] + ] + ], + "outputs": [ + [ + [ + 6.53125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.26.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 45.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.26.feed_forward.down_proj": { + "inputs": [ + [ + [ + 22.625 + ] + ] + ], + "outputs": [ + [ + [ + 22.125 + ] + ], + [ + [ + 113.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.734375 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.25 + ] + ] + ], + "outputs": [ + [ + [ + 60.25 + ] + ], + [ + [ + 71.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1953125 + ] + ], + [ + [ + 15.0625 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.5625 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1953125 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.84375 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.0625 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.5625 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.5625 + ] + ], + [ + [ + 15.0625 + ] + ], + [ + [ + 13.3125 + ] + ] + ], + "outputs": [ + [ + [ + 7.25 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.27.feed_forward.router": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "outputs": [ + [ + [ + 9.4375 + ] + ], + [ + [ + 49.25 + ] + ], + [ + [ + 131.0 + ] + ], + [ + [ + 106.5 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 50.75 + ] + ], + [ + [ + 31.75 + ] + ], + [ + [ + 89.0 + ] + ], + [ + [ + 51.0 + ] + ], + [ + [ + 41.75 + ] + ], + [ + [ + 67.0 + ] + ], + [ + [ + 96.0 + ] + ], + [ + [ + 64.5 + ] + ], + [ + [ + 49.25 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 76.0 + ] + ], + [ + [ + 38.5 + ] + ] + ] + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.453125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.283203125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "outputs": [ + [ + [ + 53.25 + ] + ], + [ + [ + 1.2428281760588963e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.87109375 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 78.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.25 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 97.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.6328125 + ] + ], + [ + [ + 17.25 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.25 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.6328125 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.671875 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.25 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.25 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 19.375 + ] + ], + [ + [ + 17.25 + ] + ], + [ + [ + 7.25 + ] + ] + ], + "outputs": [ + [ + [ + 4.25 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.28.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.349609375 + ] + ] + } + }, + "language_model.model.layers.28.feed_forward.down_proj": { + "inputs": [ + [ + [ + 42.25 + ] + ] + ], + "outputs": [ + [ + [ + 140.0 + ] + ], + [ + [ + 149.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.390625 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 67.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.1875 + ] + ] + ], + "outputs": [ + [ + [ + 78.0 + ] + ], + [ + [ + 84.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2265625 + ] + ], + [ + [ + 21.375 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.21875 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2265625 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.65625 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 21.375 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.21875 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.6875 + ] + ], + [ + [ + 21.125 + ] + ], + [ + [ + 7.21875 + ] + ] + ], + "outputs": [ + [ + [ + 5.1875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.29.feed_forward.router": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.333984375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "outputs": [ + [ + [ + 12.1875 + ] + ], + [ + [ + 108.5 + ] + ], + [ + [ + 87.5 + ] + ], + [ + [ + 50.0 + ] + ], + [ + [ + 54.0 + ] + ], + [ + [ + 64.0 + ] + ], + [ + [ + 78.0 + ] + ], + [ + [ + 74.0 + ] + ], + [ + [ + 113.5 + ] + ], + [ + [ + 94.5 + ] + ], + [ + [ + 53.25 + ] + ], + [ + [ + 82.0 + ] + ], + [ + [ + 29.25 + ] + ], + [ + [ + 48.75 + ] + ], + [ + [ + 55.0 + ] + ], + [ + [ + 54.25 + ] + ], + [ + [ + 166.0 + ] + ] + ] + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11767578125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7421875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23828125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.447265625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 77.0 + ] + ] + ], + "outputs": [ + [ + [ + 110.0 + ] + ], + [ + [ + 8.174752174077233e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.875 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.59375 + ] + ] + ], + "outputs": [ + [ + [ + 50.25 + ] + ], + [ + [ + 56.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.59375 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.21875 + ] + ], + [ + [ + 20.0 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.125 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.21875 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.59375 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.0 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.125 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.75 + ] + ], + [ + [ + 19.875 + ] + ], + [ + [ + 7.125 + ] + ] + ], + "outputs": [ + [ + [ + 4.59375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.30.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 32.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5078125 + ] + ] + } + }, + "language_model.model.layers.30.feed_forward.down_proj": { + "inputs": [ + [ + [ + 39.0 + ] + ] + ], + "outputs": [ + [ + [ + 30.25 + ] + ], + [ + [ + 159.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.447265625 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.65625 + ] + ] + ], + "outputs": [ + [ + [ + 77.0 + ] + ], + [ + [ + 115.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1640625 + ] + ], + [ + [ + 11.4375 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.5625 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1640625 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.46875 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 11.4375 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.5625 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.75 + ] + ], + [ + [ + 11.4375 + ] + ], + [ + [ + 10.5625 + ] + ] + ], + "outputs": [ + [ + [ + 6.65625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.31.feed_forward.router": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "outputs": [ + [ + [ + 25.875 + ] + ], + [ + [ + 63.25 + ] + ], + [ + [ + 71.5 + ] + ], + [ + [ + 72.5 + ] + ], + [ + [ + 67.5 + ] + ], + [ + [ + 107.5 + ] + ], + [ + [ + 75.0 + ] + ], + [ + [ + 129.0 + ] + ], + [ + [ + 132.0 + ] + ], + [ + [ + 96.0 + ] + ], + [ + [ + 117.5 + ] + ], + [ + [ + 106.0 + ] + ], + [ + [ + 171.0 + ] + ], + [ + [ + 51.75 + ] + ], + [ + [ + 38.0 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 33.5 + ] + ] + ] + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.287109375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10498046875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.369140625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.369140625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.46484375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34765625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11767578125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11279296875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.404296875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 56.5 + ] + ] + ], + "outputs": [ + [ + [ + 12.875 + ] + ], + [ + [ + 2.9110093107689658e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.53515625 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 53.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.15625 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 109.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.421875 + ] + ], + [ + [ + 18.25 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.625 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.421875 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.59375 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.25 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.625 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 17.75 + ] + ], + [ + [ + 18.25 + ] + ], + [ + [ + 6.625 + ] + ] + ], + "outputs": [ + [ + [ + 5.15625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.32.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 32.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.32.feed_forward.down_proj": { + "inputs": [ + [ + [ + 55.5 + ] + ] + ], + "outputs": [ + [ + [ + 29.375 + ] + ], + [ + [ + 75.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.453125 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 44.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.o_proj": { + "inputs": [ + [ + [ + 10.4375 + ] + ] + ], + "outputs": [ + [ + [ + 94.0 + ] + ], + [ + [ + 132.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2265625 + ] + ], + [ + [ + 15.5 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.625 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2265625 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.1875 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.5 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.625 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.3125 + ] + ], + [ + [ + 15.5 + ] + ], + [ + [ + 11.625 + ] + ] + ], + "outputs": [ + [ + [ + 10.4375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.33.feed_forward.router": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "outputs": [ + [ + [ + 15.8125 + ] + ], + [ + [ + 98.0 + ] + ], + [ + [ + 247.0 + ] + ], + [ + [ + 57.75 + ] + ], + [ + [ + 109.5 + ] + ], + [ + [ + 88.0 + ] + ], + [ + [ + 45.25 + ] + ], + [ + [ + 83.0 + ] + ], + [ + [ + 60.75 + ] + ], + [ + [ + 123.5 + ] + ], + [ + [ + 53.25 + ] + ], + [ + [ + 123.0 + ] + ], + [ + [ + 120.5 + ] + ], + [ + [ + 72.0 + ] + ], + [ + [ + 154.0 + ] + ], + [ + [ + 100.0 + ] + ], + [ + [ + 92.0 + ] + ] + ] + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.384765625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.423828125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08837890625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.36328125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 66.5 + ] + ] + ], + "outputs": [ + [ + [ + 12.75 + ] + ], + [ + [ + 1.502027635236955e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.48828125 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 54.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.59375 + ] + ] + ], + "outputs": [ + [ + [ + 55.75 + ] + ], + [ + [ + 98.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.99609375 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.4140625 + ] + ], + [ + [ + 17.875 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.9375 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4140625 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.0 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.875 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.9375 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.625 + ] + ], + [ + [ + 17.875 + ] + ], + [ + [ + 7.9375 + ] + ] + ], + "outputs": [ + [ + [ + 7.59375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.34.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6640625 + ] + ] + } + }, + "language_model.model.layers.34.feed_forward.down_proj": { + "inputs": [ + [ + [ + 75.0 + ] + ] + ], + "outputs": [ + [ + [ + 36.5 + ] + ], + [ + [ + 162.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.671875 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 54.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.9375 + ] + ] + ], + "outputs": [ + [ + [ + 110.5 + ] + ], + [ + [ + 116.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.6484375 + ] + ], + [ + [ + 13.875 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.375 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.6484375 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.78125 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.875 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 19.125 + ] + ], + [ + [ + 13.875 + ] + ], + [ + [ + 11.375 + ] + ] + ], + "outputs": [ + [ + [ + 8.9375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.35.feed_forward.router": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "outputs": [ + [ + [ + 46.0 + ] + ], + [ + [ + 115.0 + ] + ], + [ + [ + 107.0 + ] + ], + [ + [ + 348.0 + ] + ], + [ + [ + 110.5 + ] + ], + [ + [ + 115.5 + ] + ], + [ + [ + 57.75 + ] + ], + [ + [ + 81.5 + ] + ], + [ + [ + 136.0 + ] + ], + [ + [ + 63.5 + ] + ], + [ + [ + 88.0 + ] + ], + [ + [ + 127.0 + ] + ], + [ + [ + 88.5 + ] + ], + [ + [ + 88.0 + ] + ], + [ + [ + 111.0 + ] + ], + [ + [ + 168.0 + ] + ], + [ + [ + 89.5 + ] + ] + ] + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.490234375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.365234375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.54296875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33203125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.408203125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.48046875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 102.0 + ] + ] + ], + "outputs": [ + [ + [ + 26.125 + ] + ], + [ + [ + 1.502027635236955e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.51171875 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 56.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.25 + ] + ] + ], + "outputs": [ + [ + [ + 36.75 + ] + ], + [ + [ + 57.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.79296875 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.203125 + ] + ], + [ + [ + 19.0 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.875 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.203125 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.28125 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.0 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.875 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 17.75 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 9.875 + ] + ] + ], + "outputs": [ + [ + [ + 8.25 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.36.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 29.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.36.feed_forward.down_proj": { + "inputs": [ + [ + [ + 96.0 + ] + ] + ], + "outputs": [ + [ + [ + 43.0 + ] + ], + [ + [ + 94.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.59375 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.1875 + ] + ] + ], + "outputs": [ + [ + [ + 97.5 + ] + ], + [ + [ + 150.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.330078125 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1953125 + ] + ], + [ + [ + 23.375 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.625 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1953125 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.40625 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.625 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.375 + ] + ], + [ + [ + 23.375 + ] + ], + [ + [ + 8.625 + ] + ] + ], + "outputs": [ + [ + [ + 8.1875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.37.feed_forward.router": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "outputs": [ + [ + [ + 27.875 + ] + ], + [ + [ + 133.0 + ] + ], + [ + [ + 114.0 + ] + ], + [ + [ + 126.5 + ] + ], + [ + [ + 123.0 + ] + ], + [ + [ + 140.0 + ] + ], + [ + [ + 81.5 + ] + ], + [ + [ + 116.5 + ] + ], + [ + [ + 55.0 + ] + ], + [ + [ + 129.0 + ] + ], + [ + [ + 119.0 + ] + ], + [ + [ + 105.5 + ] + ], + [ + [ + 114.0 + ] + ], + [ + [ + 170.0 + ] + ], + [ + [ + 142.0 + ] + ], + [ + [ + 96.5 + ] + ], + [ + [ + 143.0 + ] + ] + ] + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.396484375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.310546875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.51171875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 124.5 + ] + ] + ], + "outputs": [ + [ + [ + 83.5 + ] + ], + [ + [ + 2.2331030329186587e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.78125 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 47.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.o_proj": { + "inputs": [ + [ + [ + 10.4375 + ] + ] + ], + "outputs": [ + [ + [ + 29.75 + ] + ], + [ + [ + 66.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.21875 + ] + ], + [ + [ + 22.5 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.875 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.21875 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.625 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 22.5 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.875 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.25 + ] + ], + [ + [ + 21.125 + ] + ], + [ + [ + 11.875 + ] + ] + ], + "outputs": [ + [ + [ + 10.4375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.38.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 31.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5078125 + ] + ] + } + }, + "language_model.model.layers.38.feed_forward.down_proj": { + "inputs": [ + [ + [ + 109.5 + ] + ] + ], + "outputs": [ + [ + [ + 31.125 + ] + ], + [ + [ + 262.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.52734375 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 52.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.o_proj": { + "inputs": [ + [ + [ + 13.0 + ] + ] + ], + "outputs": [ + [ + [ + 126.5 + ] + ], + [ + [ + 154.0 + ] + ] + ], + "params": { + "weight": [ + [ + 1.4140625 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.828125 + ] + ], + [ + [ + 9.5625 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 15.4375 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.828125 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.9375 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 9.5625 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 15.4375 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.0625 + ] + ], + [ + [ + 9.5625 + ] + ], + [ + [ + 15.4375 + ] + ] + ], + "outputs": [ + [ + [ + 13.0 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.39.feed_forward.router": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "outputs": [ + [ + [ + 39.0 + ] + ], + [ + [ + 124.5 + ] + ], + [ + [ + 169.0 + ] + ], + [ + [ + 88.0 + ] + ], + [ + [ + 124.5 + ] + ], + [ + [ + 77.0 + ] + ], + [ + [ + 133.0 + ] + ], + [ + [ + 68.0 + ] + ], + [ + [ + 115.5 + ] + ], + [ + [ + 129.0 + ] + ], + [ + [ + 176.0 + ] + ], + [ + [ + 155.0 + ] + ], + [ + [ + 135.0 + ] + ], + [ + [ + 116.5 + ] + ], + [ + [ + 161.0 + ] + ], + [ + [ + 146.0 + ] + ], + [ + [ + 114.5 + ] + ] + ] + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2490234375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.322265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.404296875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 164.0 + ] + ] + ], + "outputs": [ + [ + [ + 52.5 + ] + ], + [ + [ + 1.2428281760588963e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.82421875 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 46.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.o_proj": { + "inputs": [ + [ + [ + 10.4375 + ] + ] + ], + "outputs": [ + [ + [ + 94.0 + ] + ], + [ + [ + 102.0 + ] + ] + ], + "params": { + "weight": [ + [ + 1.15625 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1640625 + ] + ], + [ + [ + 18.25 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.625 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1640625 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.125 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.25 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.625 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.4375 + ] + ], + [ + [ + 18.25 + ] + ], + [ + [ + 11.625 + ] + ] + ], + "outputs": [ + [ + [ + 10.4375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.40.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.40.feed_forward.down_proj": { + "inputs": [ + [ + [ + 142.0 + ] + ] + ], + "outputs": [ + [ + [ + 28.5 + ] + ], + [ + [ + 127.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.53515625 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 35.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.o_proj": { + "inputs": [ + [ + [ + 14.6875 + ] + ] + ], + "outputs": [ + [ + [ + 160.0 + ] + ], + [ + [ + 177.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40625 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.8203125 + ] + ], + [ + [ + 13.75 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 15.375 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.8203125 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.1875 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.75 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 15.375 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 22.75 + ] + ], + [ + [ + 13.75 + ] + ], + [ + [ + 15.375 + ] + ] + ], + "outputs": [ + [ + [ + 14.6875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.41.feed_forward.router": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "outputs": [ + [ + [ + 54.0 + ] + ], + [ + [ + 179.0 + ] + ], + [ + [ + 103.5 + ] + ], + [ + [ + 145.0 + ] + ], + [ + [ + 141.0 + ] + ], + [ + [ + 266.0 + ] + ], + [ + [ + 109.0 + ] + ], + [ + [ + 233.0 + ] + ], + [ + [ + 180.0 + ] + ], + [ + [ + 216.0 + ] + ], + [ + [ + 159.0 + ] + ], + [ + [ + 382.0 + ] + ], + [ + [ + 89.5 + ] + ], + [ + [ + 218.0 + ] + ], + [ + [ + 152.0 + ] + ], + [ + [ + 155.0 + ] + ], + [ + [ + 268.0 + ] + ] + ] + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.39453125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1083984375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.435546875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.330078125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4609375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 191.0 + ] + ] + ], + "outputs": [ + [ + [ + 34.75 + ] + ], + [ + [ + 2.2331030329186587e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.625 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 39.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.o_proj": { + "inputs": [ + [ + [ + 12.5625 + ] + ] + ], + "outputs": [ + [ + [ + 65.0 + ] + ], + [ + [ + 146.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.609375 + ] + ], + [ + [ + 12.1875 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 16.625 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.609375 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.0625 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 12.1875 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 18.5 + ] + ], + [ + [ + 11.9375 + ] + ], + [ + [ + 16.625 + ] + ] + ], + "outputs": [ + [ + [ + 12.5625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.42.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 43.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.349609375 + ] + ] + } + }, + "language_model.model.layers.42.feed_forward.down_proj": { + "inputs": [ + [ + [ + 152.0 + ] + ] + ], + "outputs": [ + [ + [ + 70.5 + ] + ], + [ + [ + 218.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.41015625 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 43.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.o_proj": { + "inputs": [ + [ + [ + 12.75 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 232.0 + ] + ] + ], + "params": { + "weight": [ + [ + 1.0625 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.3828125 + ] + ], + [ + [ + 13.0625 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.75 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.3828125 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.6875 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.0625 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 12.75 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.875 + ] + ], + [ + [ + 13.0625 + ] + ], + [ + [ + 12.75 + ] + ] + ], + "outputs": [ + [ + [ + 12.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.43.feed_forward.router": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "outputs": [ + [ + [ + 87.5 + ] + ], + [ + [ + 148.0 + ] + ], + [ + [ + 180.0 + ] + ], + [ + [ + 348.0 + ] + ], + [ + [ + 456.0 + ] + ], + [ + [ + 239.0 + ] + ], + [ + [ + 145.0 + ] + ], + [ + [ + 228.0 + ] + ], + [ + [ + 148.0 + ] + ], + [ + [ + 126.5 + ] + ], + [ + [ + 197.0 + ] + ], + [ + [ + 159.0 + ] + ], + [ + [ + 86.5 + ] + ], + [ + [ + 292.0 + ] + ], + [ + [ + 164.0 + ] + ], + [ + [ + 136.0 + ] + ], + [ + [ + 175.0 + ] + ] + ] + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.49609375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4140625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 202.0 + ] + ] + ], + "outputs": [ + [ + [ + 27.375 + ] + ], + [ + [ + 2.9110093107689658e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.52734375 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 29.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.o_proj": { + "inputs": [ + [ + [ + 14.9375 + ] + ] + ], + "outputs": [ + [ + [ + 81.0 + ] + ], + [ + [ + 170.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8828125 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 2.171875 + ] + ], + [ + [ + 20.125 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 15.1875 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.171875 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.9375 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.125 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 15.1875 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 24.125 + ] + ], + [ + [ + 20.125 + ] + ], + [ + [ + 14.5 + ] + ] + ], + "outputs": [ + [ + [ + 13.9375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.44.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 42.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.44.feed_forward.down_proj": { + "inputs": [ + [ + [ + 218.0 + ] + ] + ], + "outputs": [ + [ + [ + 75.5 + ] + ], + [ + [ + 200.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.419921875 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 28.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.o_proj": { + "inputs": [ + [ + [ + 31.375 + ] + ] + ], + "outputs": [ + [ + [ + 170.0 + ] + ], + [ + [ + 258.0 + ] + ] + ], + "params": { + "weight": [ + [ + 1.1640625 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.9921875 + ] + ], + [ + [ + 15.6875 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 31.625 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.9921875 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 27.125 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.6875 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 31.625 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 22.5 + ] + ], + [ + [ + 15.6875 + ] + ], + [ + [ + 31.625 + ] + ] + ], + "outputs": [ + [ + [ + 31.375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.45.feed_forward.router": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "outputs": [ + [ + [ + 23.125 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 59.5 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 97.5 + ] + ], + [ + [ + 45.5 + ] + ], + [ + [ + 128.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ] + ] + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.59765625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11181640625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.412109375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2353515625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09326171875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33203125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 510.0 + ] + ] + ], + "outputs": [ + [ + [ + 53.0 + ] + ], + [ + [ + 2.9110093107689658e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.369140625 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 27.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.32421875 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.o_proj": { + "inputs": [ + [ + [ + 22.5 + ] + ] + ], + "outputs": [ + [ + [ + 126.0 + ] + ], + [ + [ + 221.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.287109375 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 2.171875 + ] + ], + [ + [ + 19.75 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 22.5 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.171875 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 20.375 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.75 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 22.5 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 29.875 + ] + ], + [ + [ + 19.75 + ] + ], + [ + [ + 22.5 + ] + ] + ], + "outputs": [ + [ + [ + 22.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.46.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 40.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.46.feed_forward.down_proj": { + "inputs": [ + [ + [ + 576.0 + ] + ] + ], + "outputs": [ + [ + [ + 1680.0 + ] + ], + [ + [ + 2400.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.65234375 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.0625 + ] + ] + ], + "outputs": [ + [ + [ + 96.5 + ] + ], + [ + [ + 125.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.408203125 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.50390625 + ] + ], + [ + [ + 6.4375 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.75 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.50390625 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.8125 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 6.4375 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.75 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 5.75 + ] + ], + [ + [ + 6.4375 + ] + ], + [ + [ + 10.5 + ] + ] + ], + "outputs": [ + [ + [ + 8.0625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.47.feed_forward.router": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "outputs": [ + [ + [ + 708.0 + ] + ], + [ + [ + 109.5 + ] + ], + [ + [ + 146.0 + ] + ], + [ + [ + 108.5 + ] + ], + [ + [ + 202.0 + ] + ], + [ + [ + 320.0 + ] + ], + [ + [ + 187.0 + ] + ], + [ + [ + 247.0 + ] + ], + [ + [ + 112.5 + ] + ], + [ + [ + 316.0 + ] + ], + [ + [ + 764.0 + ] + ], + [ + [ + 202.0 + ] + ], + [ + [ + 205.0 + ] + ], + [ + [ + 93.0 + ] + ], + [ + [ + 154.0 + ] + ], + [ + [ + 102.0 + ] + ], + [ + [ + 0.0 + ] + ] + ] + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.361328125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.515625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.65234375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7265625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.91796875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.9140625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7265625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.498046875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.67578125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.74609375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.859375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4765625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.84375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6015625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6015625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 1.046875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.59375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 840.0 + ] + ] + ], + "outputs": [ + [ + [ + 940.0 + ] + ], + [ + [ + 2.9110093107689658e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.416015625 + ] + ] + } + }, + "language_model.lm_head": { + "inputs": [ + [ + [ + 54.25 + ] + ] + ], + "params": { + "weight": [ + [ + 1.1875 + ] + ] + } + } + } +} \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_1_8.npz b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_1_8.npz new file mode 100644 index 000000000000..6f23217022e2 Binary files /dev/null and b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_1_8.npz differ diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_1_8_mod_list.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_1_8_mod_list.json new file mode 100644 index 000000000000..449fce78b77d --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_1_8_mod_list.json @@ -0,0 +1,1559 @@ +[ + "vision_model.patch_embedding.linear", + "vision_model.model.layers.0.self_attn.qkv_proj", + "vision_model.model.layers.0.self_attn.o_proj", + "vision_model.model.layers.0.mlp.fc1", + "vision_model.model.layers.0.mlp.fc2", + "vision_model.model.layers.1.self_attn.qkv_proj", + "vision_model.model.layers.1.self_attn.o_proj", + "vision_model.model.layers.1.mlp.fc1", + "vision_model.model.layers.1.mlp.fc2", + "vision_model.model.layers.2.self_attn.qkv_proj", + "vision_model.model.layers.2.self_attn.o_proj", + "vision_model.model.layers.2.mlp.fc1", + "vision_model.model.layers.2.mlp.fc2", + "vision_model.model.layers.3.self_attn.qkv_proj", + "vision_model.model.layers.3.self_attn.o_proj", + "vision_model.model.layers.3.mlp.fc1", + "vision_model.model.layers.3.mlp.fc2", + "vision_model.model.layers.4.self_attn.qkv_proj", + "vision_model.model.layers.4.self_attn.o_proj", + "vision_model.model.layers.4.mlp.fc1", + "vision_model.model.layers.4.mlp.fc2", + "vision_model.model.layers.5.self_attn.qkv_proj", + "vision_model.model.layers.5.self_attn.o_proj", + "vision_model.model.layers.5.mlp.fc1", + "vision_model.model.layers.5.mlp.fc2", + "vision_model.model.layers.6.self_attn.qkv_proj", + "vision_model.model.layers.6.self_attn.o_proj", + "vision_model.model.layers.6.mlp.fc1", + "vision_model.model.layers.6.mlp.fc2", + "vision_model.model.layers.7.self_attn.qkv_proj", + "vision_model.model.layers.7.self_attn.o_proj", + "vision_model.model.layers.7.mlp.fc1", + "vision_model.model.layers.7.mlp.fc2", + "vision_model.model.layers.8.self_attn.qkv_proj", + "vision_model.model.layers.8.self_attn.o_proj", + "vision_model.model.layers.8.mlp.fc1", + "vision_model.model.layers.8.mlp.fc2", + "vision_model.model.layers.9.self_attn.qkv_proj", + "vision_model.model.layers.9.self_attn.o_proj", + "vision_model.model.layers.9.mlp.fc1", + "vision_model.model.layers.9.mlp.fc2", + "vision_model.model.layers.10.self_attn.qkv_proj", + "vision_model.model.layers.10.self_attn.o_proj", + "vision_model.model.layers.10.mlp.fc1", + "vision_model.model.layers.10.mlp.fc2", + "vision_model.model.layers.11.self_attn.qkv_proj", + "vision_model.model.layers.11.self_attn.o_proj", + "vision_model.model.layers.11.mlp.fc1", + "vision_model.model.layers.11.mlp.fc2", + "vision_model.model.layers.12.self_attn.qkv_proj", + "vision_model.model.layers.12.self_attn.o_proj", + "vision_model.model.layers.12.mlp.fc1", + "vision_model.model.layers.12.mlp.fc2", + "vision_model.model.layers.13.self_attn.qkv_proj", + "vision_model.model.layers.13.self_attn.o_proj", + "vision_model.model.layers.13.mlp.fc1", + "vision_model.model.layers.13.mlp.fc2", + "vision_model.model.layers.14.self_attn.qkv_proj", + "vision_model.model.layers.14.self_attn.o_proj", + "vision_model.model.layers.14.mlp.fc1", + "vision_model.model.layers.14.mlp.fc2", + "vision_model.model.layers.15.self_attn.qkv_proj", + "vision_model.model.layers.15.self_attn.o_proj", + "vision_model.model.layers.15.mlp.fc1", + "vision_model.model.layers.15.mlp.fc2", + "vision_model.model.layers.16.self_attn.qkv_proj", + "vision_model.model.layers.16.self_attn.o_proj", + "vision_model.model.layers.16.mlp.fc1", + "vision_model.model.layers.16.mlp.fc2", + "vision_model.model.layers.17.self_attn.qkv_proj", + "vision_model.model.layers.17.self_attn.o_proj", + "vision_model.model.layers.17.mlp.fc1", + "vision_model.model.layers.17.mlp.fc2", + "vision_model.model.layers.18.self_attn.qkv_proj", + "vision_model.model.layers.18.self_attn.o_proj", + "vision_model.model.layers.18.mlp.fc1", + "vision_model.model.layers.18.mlp.fc2", + "vision_model.model.layers.19.self_attn.qkv_proj", + "vision_model.model.layers.19.self_attn.o_proj", + "vision_model.model.layers.19.mlp.fc1", + "vision_model.model.layers.19.mlp.fc2", + "vision_model.model.layers.20.self_attn.qkv_proj", + "vision_model.model.layers.20.self_attn.o_proj", + "vision_model.model.layers.20.mlp.fc1", + "vision_model.model.layers.20.mlp.fc2", + "vision_model.model.layers.21.self_attn.qkv_proj", + "vision_model.model.layers.21.self_attn.o_proj", + "vision_model.model.layers.21.mlp.fc1", + "vision_model.model.layers.21.mlp.fc2", + "vision_model.model.layers.22.self_attn.qkv_proj", + "vision_model.model.layers.22.self_attn.o_proj", + "vision_model.model.layers.22.mlp.fc1", + "vision_model.model.layers.22.mlp.fc2", + "vision_model.model.layers.23.self_attn.qkv_proj", + "vision_model.model.layers.23.self_attn.o_proj", + "vision_model.model.layers.23.mlp.fc1", + "vision_model.model.layers.23.mlp.fc2", + "vision_model.model.layers.24.self_attn.qkv_proj", + "vision_model.model.layers.24.self_attn.o_proj", + "vision_model.model.layers.24.mlp.fc1", + "vision_model.model.layers.24.mlp.fc2", + "vision_model.model.layers.25.self_attn.qkv_proj", + "vision_model.model.layers.25.self_attn.o_proj", + "vision_model.model.layers.25.mlp.fc1", + "vision_model.model.layers.25.mlp.fc2", + "vision_model.model.layers.26.self_attn.qkv_proj", + "vision_model.model.layers.26.self_attn.o_proj", + "vision_model.model.layers.26.mlp.fc1", + "vision_model.model.layers.26.mlp.fc2", + "vision_model.model.layers.27.self_attn.qkv_proj", + "vision_model.model.layers.27.self_attn.o_proj", + "vision_model.model.layers.27.mlp.fc1", + "vision_model.model.layers.27.mlp.fc2", + "vision_model.model.layers.28.self_attn.qkv_proj", + "vision_model.model.layers.28.self_attn.o_proj", + "vision_model.model.layers.28.mlp.fc1", + "vision_model.model.layers.28.mlp.fc2", + "vision_model.model.layers.29.self_attn.qkv_proj", + "vision_model.model.layers.29.self_attn.o_proj", + "vision_model.model.layers.29.mlp.fc1", + "vision_model.model.layers.29.mlp.fc2", + "vision_model.model.layers.30.self_attn.qkv_proj", + "vision_model.model.layers.30.self_attn.o_proj", + "vision_model.model.layers.30.mlp.fc1", + "vision_model.model.layers.30.mlp.fc2", + "vision_model.model.layers.31.self_attn.qkv_proj", + "vision_model.model.layers.31.self_attn.o_proj", + "vision_model.model.layers.31.mlp.fc1", + "vision_model.model.layers.31.mlp.fc2", + "vision_model.model.layers.32.self_attn.qkv_proj", + "vision_model.model.layers.32.self_attn.o_proj", + "vision_model.model.layers.32.mlp.fc1", + "vision_model.model.layers.32.mlp.fc2", + "vision_model.model.layers.33.self_attn.qkv_proj", + "vision_model.model.layers.33.self_attn.o_proj", + "vision_model.model.layers.33.mlp.fc1", + "vision_model.model.layers.33.mlp.fc2", + "vision_model.vision_adapter.mlp.fc1", + "vision_model.vision_adapter.mlp.fc2", + "multi_modal_projector.linear_1", + "language_model.model.layers.0.self_attn.qkv_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.0.self_attn.attn.impl.softmax", + "language_model.model.layers.0.self_attn.attn.impl.matmul_av", + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.0.self_attn.attn.impl.k_cache", + "language_model.model.layers.0.self_attn.attn.impl.v_cache", + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.0.feed_forward.gate_up_proj", + "language_model.model.layers.0.feed_forward.down_proj", + "language_model.model.layers.1.self_attn.qkv_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.1.self_attn.attn.impl.softmax", + "language_model.model.layers.1.self_attn.attn.impl.matmul_av", + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.1.self_attn.attn.impl.k_cache", + "language_model.model.layers.1.self_attn.attn.impl.v_cache", + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.1.feed_forward.router", + "language_model.model.layers.1.feed_forward.experts", + "language_model.model.layers.1.feed_forward.experts.moe_op", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.1.feed_forward.shared_expert.down_proj", + "language_model.model.layers.2.self_attn.qkv_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.2.self_attn.attn.impl.softmax", + "language_model.model.layers.2.self_attn.attn.impl.matmul_av", + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.2.self_attn.attn.impl.k_cache", + "language_model.model.layers.2.self_attn.attn.impl.v_cache", + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.2.feed_forward.gate_up_proj", + "language_model.model.layers.2.feed_forward.down_proj", + "language_model.model.layers.3.self_attn.qkv_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.3.self_attn.attn.impl.softmax", + "language_model.model.layers.3.self_attn.attn.impl.matmul_av", + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.3.self_attn.attn.impl.k_cache", + "language_model.model.layers.3.self_attn.attn.impl.v_cache", + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.3.feed_forward.router", + "language_model.model.layers.3.feed_forward.experts", + "language_model.model.layers.3.feed_forward.experts.moe_op", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.3.feed_forward.shared_expert.down_proj", + "language_model.model.layers.4.self_attn.qkv_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.4.self_attn.attn.impl.softmax", + "language_model.model.layers.4.self_attn.attn.impl.matmul_av", + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.4.self_attn.attn.impl.k_cache", + "language_model.model.layers.4.self_attn.attn.impl.v_cache", + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.4.feed_forward.gate_up_proj", + "language_model.model.layers.4.feed_forward.down_proj", + "language_model.model.layers.5.self_attn.qkv_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.5.self_attn.attn.impl.softmax", + "language_model.model.layers.5.self_attn.attn.impl.matmul_av", + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.5.self_attn.attn.impl.k_cache", + "language_model.model.layers.5.self_attn.attn.impl.v_cache", + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.5.feed_forward.router", + "language_model.model.layers.5.feed_forward.experts", + "language_model.model.layers.5.feed_forward.experts.moe_op", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.5.feed_forward.shared_expert.down_proj", + "language_model.model.layers.6.self_attn.qkv_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.6.self_attn.attn.impl.softmax", + "language_model.model.layers.6.self_attn.attn.impl.matmul_av", + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.6.self_attn.attn.impl.k_cache", + "language_model.model.layers.6.self_attn.attn.impl.v_cache", + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.6.feed_forward.gate_up_proj", + "language_model.model.layers.6.feed_forward.down_proj", + "language_model.model.layers.7.self_attn.qkv_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.7.self_attn.attn.impl.softmax", + "language_model.model.layers.7.self_attn.attn.impl.matmul_av", + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.7.self_attn.attn.impl.k_cache", + "language_model.model.layers.7.self_attn.attn.impl.v_cache", + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.7.feed_forward.router", + "language_model.model.layers.7.feed_forward.experts", + "language_model.model.layers.7.feed_forward.experts.moe_op", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.7.feed_forward.shared_expert.down_proj", + "language_model.model.layers.8.self_attn.qkv_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.8.self_attn.attn.impl.softmax", + "language_model.model.layers.8.self_attn.attn.impl.matmul_av", + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.8.self_attn.attn.impl.k_cache", + "language_model.model.layers.8.self_attn.attn.impl.v_cache", + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.8.feed_forward.gate_up_proj", + "language_model.model.layers.8.feed_forward.down_proj", + "language_model.model.layers.9.self_attn.qkv_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.9.self_attn.attn.impl.softmax", + "language_model.model.layers.9.self_attn.attn.impl.matmul_av", + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.9.self_attn.attn.impl.k_cache", + "language_model.model.layers.9.self_attn.attn.impl.v_cache", + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.9.feed_forward.router", + "language_model.model.layers.9.feed_forward.experts", + "language_model.model.layers.9.feed_forward.experts.moe_op", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.9.feed_forward.shared_expert.down_proj", + "language_model.model.layers.10.self_attn.qkv_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.10.self_attn.attn.impl.softmax", + "language_model.model.layers.10.self_attn.attn.impl.matmul_av", + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.10.self_attn.attn.impl.k_cache", + "language_model.model.layers.10.self_attn.attn.impl.v_cache", + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.10.feed_forward.gate_up_proj", + "language_model.model.layers.10.feed_forward.down_proj", + "language_model.model.layers.11.self_attn.qkv_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.11.self_attn.attn.impl.softmax", + "language_model.model.layers.11.self_attn.attn.impl.matmul_av", + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.11.self_attn.attn.impl.k_cache", + "language_model.model.layers.11.self_attn.attn.impl.v_cache", + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.11.feed_forward.router", + "language_model.model.layers.11.feed_forward.experts", + "language_model.model.layers.11.feed_forward.experts.moe_op", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.11.feed_forward.shared_expert.down_proj", + "language_model.model.layers.12.self_attn.qkv_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.12.self_attn.attn.impl.softmax", + "language_model.model.layers.12.self_attn.attn.impl.matmul_av", + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.12.self_attn.attn.impl.k_cache", + "language_model.model.layers.12.self_attn.attn.impl.v_cache", + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.12.feed_forward.gate_up_proj", + "language_model.model.layers.12.feed_forward.down_proj", + "language_model.model.layers.13.self_attn.qkv_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.13.self_attn.attn.impl.softmax", + "language_model.model.layers.13.self_attn.attn.impl.matmul_av", + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.13.self_attn.attn.impl.k_cache", + "language_model.model.layers.13.self_attn.attn.impl.v_cache", + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.13.feed_forward.router", + "language_model.model.layers.13.feed_forward.experts", + "language_model.model.layers.13.feed_forward.experts.moe_op", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.13.feed_forward.shared_expert.down_proj", + "language_model.model.layers.14.self_attn.qkv_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.14.self_attn.attn.impl.softmax", + "language_model.model.layers.14.self_attn.attn.impl.matmul_av", + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.14.self_attn.attn.impl.k_cache", + "language_model.model.layers.14.self_attn.attn.impl.v_cache", + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.14.feed_forward.gate_up_proj", + "language_model.model.layers.14.feed_forward.down_proj", + "language_model.model.layers.15.self_attn.qkv_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.15.self_attn.attn.impl.softmax", + "language_model.model.layers.15.self_attn.attn.impl.matmul_av", + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.15.self_attn.attn.impl.k_cache", + "language_model.model.layers.15.self_attn.attn.impl.v_cache", + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.15.feed_forward.router", + "language_model.model.layers.15.feed_forward.experts", + "language_model.model.layers.15.feed_forward.experts.moe_op", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.15.feed_forward.shared_expert.down_proj", + "language_model.model.layers.16.self_attn.qkv_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.16.self_attn.attn.impl.softmax", + "language_model.model.layers.16.self_attn.attn.impl.matmul_av", + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.16.self_attn.attn.impl.k_cache", + "language_model.model.layers.16.self_attn.attn.impl.v_cache", + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.16.feed_forward.gate_up_proj", + "language_model.model.layers.16.feed_forward.down_proj", + "language_model.model.layers.17.self_attn.qkv_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.17.self_attn.attn.impl.softmax", + "language_model.model.layers.17.self_attn.attn.impl.matmul_av", + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.17.self_attn.attn.impl.k_cache", + "language_model.model.layers.17.self_attn.attn.impl.v_cache", + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.17.feed_forward.router", + "language_model.model.layers.17.feed_forward.experts", + "language_model.model.layers.17.feed_forward.experts.moe_op", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.17.feed_forward.shared_expert.down_proj", + "language_model.model.layers.18.self_attn.qkv_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.18.self_attn.attn.impl.softmax", + "language_model.model.layers.18.self_attn.attn.impl.matmul_av", + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.18.self_attn.attn.impl.k_cache", + "language_model.model.layers.18.self_attn.attn.impl.v_cache", + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.18.feed_forward.gate_up_proj", + "language_model.model.layers.18.feed_forward.down_proj", + "language_model.model.layers.19.self_attn.qkv_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.19.self_attn.attn.impl.softmax", + "language_model.model.layers.19.self_attn.attn.impl.matmul_av", + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.19.self_attn.attn.impl.k_cache", + "language_model.model.layers.19.self_attn.attn.impl.v_cache", + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.19.feed_forward.router", + "language_model.model.layers.19.feed_forward.experts", + "language_model.model.layers.19.feed_forward.experts.moe_op", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.19.feed_forward.shared_expert.down_proj", + "language_model.model.layers.20.self_attn.qkv_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.20.self_attn.attn.impl.softmax", + "language_model.model.layers.20.self_attn.attn.impl.matmul_av", + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.20.self_attn.attn.impl.k_cache", + "language_model.model.layers.20.self_attn.attn.impl.v_cache", + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.20.feed_forward.gate_up_proj", + "language_model.model.layers.20.feed_forward.down_proj", + "language_model.model.layers.21.self_attn.qkv_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.21.self_attn.attn.impl.softmax", + "language_model.model.layers.21.self_attn.attn.impl.matmul_av", + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.21.self_attn.attn.impl.k_cache", + "language_model.model.layers.21.self_attn.attn.impl.v_cache", + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.21.feed_forward.router", + "language_model.model.layers.21.feed_forward.experts", + "language_model.model.layers.21.feed_forward.experts.moe_op", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.21.feed_forward.shared_expert.down_proj", + "language_model.model.layers.22.self_attn.qkv_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.22.self_attn.attn.impl.softmax", + "language_model.model.layers.22.self_attn.attn.impl.matmul_av", + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.22.self_attn.attn.impl.k_cache", + "language_model.model.layers.22.self_attn.attn.impl.v_cache", + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.22.feed_forward.gate_up_proj", + "language_model.model.layers.22.feed_forward.down_proj", + "language_model.model.layers.23.self_attn.qkv_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.23.self_attn.attn.impl.softmax", + "language_model.model.layers.23.self_attn.attn.impl.matmul_av", + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.23.self_attn.attn.impl.k_cache", + "language_model.model.layers.23.self_attn.attn.impl.v_cache", + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.23.feed_forward.router", + "language_model.model.layers.23.feed_forward.experts", + "language_model.model.layers.23.feed_forward.experts.moe_op", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.23.feed_forward.shared_expert.down_proj", + "language_model.model.layers.24.self_attn.qkv_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.24.self_attn.attn.impl.softmax", + "language_model.model.layers.24.self_attn.attn.impl.matmul_av", + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.24.self_attn.attn.impl.k_cache", + "language_model.model.layers.24.self_attn.attn.impl.v_cache", + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.24.feed_forward.gate_up_proj", + "language_model.model.layers.24.feed_forward.down_proj", + "language_model.model.layers.25.self_attn.qkv_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.25.self_attn.attn.impl.softmax", + "language_model.model.layers.25.self_attn.attn.impl.matmul_av", + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.25.self_attn.attn.impl.k_cache", + "language_model.model.layers.25.self_attn.attn.impl.v_cache", + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.25.feed_forward.router", + "language_model.model.layers.25.feed_forward.experts", + "language_model.model.layers.25.feed_forward.experts.moe_op", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.25.feed_forward.shared_expert.down_proj", + "language_model.model.layers.26.self_attn.qkv_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.26.self_attn.attn.impl.softmax", + "language_model.model.layers.26.self_attn.attn.impl.matmul_av", + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.26.self_attn.attn.impl.k_cache", + "language_model.model.layers.26.self_attn.attn.impl.v_cache", + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.26.feed_forward.gate_up_proj", + "language_model.model.layers.26.feed_forward.down_proj", + "language_model.model.layers.27.self_attn.qkv_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.27.self_attn.attn.impl.softmax", + "language_model.model.layers.27.self_attn.attn.impl.matmul_av", + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.27.self_attn.attn.impl.k_cache", + "language_model.model.layers.27.self_attn.attn.impl.v_cache", + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.27.feed_forward.router", + "language_model.model.layers.27.feed_forward.experts", + "language_model.model.layers.27.feed_forward.experts.moe_op", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.27.feed_forward.shared_expert.down_proj", + "language_model.model.layers.28.self_attn.qkv_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.28.self_attn.attn.impl.softmax", + "language_model.model.layers.28.self_attn.attn.impl.matmul_av", + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.28.self_attn.attn.impl.k_cache", + "language_model.model.layers.28.self_attn.attn.impl.v_cache", + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.28.feed_forward.gate_up_proj", + "language_model.model.layers.28.feed_forward.down_proj", + "language_model.model.layers.29.self_attn.qkv_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.29.self_attn.attn.impl.softmax", + "language_model.model.layers.29.self_attn.attn.impl.matmul_av", + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.29.self_attn.attn.impl.k_cache", + "language_model.model.layers.29.self_attn.attn.impl.v_cache", + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.29.feed_forward.router", + "language_model.model.layers.29.feed_forward.experts", + "language_model.model.layers.29.feed_forward.experts.moe_op", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.29.feed_forward.shared_expert.down_proj", + "language_model.model.layers.30.self_attn.qkv_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.30.self_attn.attn.impl.softmax", + "language_model.model.layers.30.self_attn.attn.impl.matmul_av", + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.30.self_attn.attn.impl.k_cache", + "language_model.model.layers.30.self_attn.attn.impl.v_cache", + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.30.feed_forward.gate_up_proj", + "language_model.model.layers.30.feed_forward.down_proj", + "language_model.model.layers.31.self_attn.qkv_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.31.self_attn.attn.impl.softmax", + "language_model.model.layers.31.self_attn.attn.impl.matmul_av", + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.31.self_attn.attn.impl.k_cache", + "language_model.model.layers.31.self_attn.attn.impl.v_cache", + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.31.feed_forward.router", + "language_model.model.layers.31.feed_forward.experts", + "language_model.model.layers.31.feed_forward.experts.moe_op", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.31.feed_forward.shared_expert.down_proj", + "language_model.model.layers.32.self_attn.qkv_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.32.self_attn.attn.impl.softmax", + "language_model.model.layers.32.self_attn.attn.impl.matmul_av", + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.32.self_attn.attn.impl.k_cache", + "language_model.model.layers.32.self_attn.attn.impl.v_cache", + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.32.feed_forward.gate_up_proj", + "language_model.model.layers.32.feed_forward.down_proj", + "language_model.model.layers.33.self_attn.qkv_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.33.self_attn.attn.impl.softmax", + "language_model.model.layers.33.self_attn.attn.impl.matmul_av", + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.33.self_attn.attn.impl.k_cache", + "language_model.model.layers.33.self_attn.attn.impl.v_cache", + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.33.feed_forward.router", + "language_model.model.layers.33.feed_forward.experts", + "language_model.model.layers.33.feed_forward.experts.moe_op", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.33.feed_forward.shared_expert.down_proj", + "language_model.model.layers.34.self_attn.qkv_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.34.self_attn.attn.impl.softmax", + "language_model.model.layers.34.self_attn.attn.impl.matmul_av", + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.34.self_attn.attn.impl.k_cache", + "language_model.model.layers.34.self_attn.attn.impl.v_cache", + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.34.feed_forward.gate_up_proj", + "language_model.model.layers.34.feed_forward.down_proj", + "language_model.model.layers.35.self_attn.qkv_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.35.self_attn.attn.impl.softmax", + "language_model.model.layers.35.self_attn.attn.impl.matmul_av", + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.35.self_attn.attn.impl.k_cache", + "language_model.model.layers.35.self_attn.attn.impl.v_cache", + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.35.feed_forward.router", + "language_model.model.layers.35.feed_forward.experts", + "language_model.model.layers.35.feed_forward.experts.moe_op", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.35.feed_forward.shared_expert.down_proj", + "language_model.model.layers.36.self_attn.qkv_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.36.self_attn.attn.impl.softmax", + "language_model.model.layers.36.self_attn.attn.impl.matmul_av", + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.36.self_attn.attn.impl.k_cache", + "language_model.model.layers.36.self_attn.attn.impl.v_cache", + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.36.feed_forward.gate_up_proj", + "language_model.model.layers.36.feed_forward.down_proj", + "language_model.model.layers.37.self_attn.qkv_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.37.self_attn.attn.impl.softmax", + "language_model.model.layers.37.self_attn.attn.impl.matmul_av", + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.37.self_attn.attn.impl.k_cache", + "language_model.model.layers.37.self_attn.attn.impl.v_cache", + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.37.feed_forward.router", + "language_model.model.layers.37.feed_forward.experts", + "language_model.model.layers.37.feed_forward.experts.moe_op", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.37.feed_forward.shared_expert.down_proj", + "language_model.model.layers.38.self_attn.qkv_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.38.self_attn.attn.impl.softmax", + "language_model.model.layers.38.self_attn.attn.impl.matmul_av", + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.38.self_attn.attn.impl.k_cache", + "language_model.model.layers.38.self_attn.attn.impl.v_cache", + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.38.feed_forward.gate_up_proj", + "language_model.model.layers.38.feed_forward.down_proj", + "language_model.model.layers.39.self_attn.qkv_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.39.self_attn.attn.impl.softmax", + "language_model.model.layers.39.self_attn.attn.impl.matmul_av", + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.39.self_attn.attn.impl.k_cache", + "language_model.model.layers.39.self_attn.attn.impl.v_cache", + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.39.feed_forward.router", + "language_model.model.layers.39.feed_forward.experts", + "language_model.model.layers.39.feed_forward.experts.moe_op", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.39.feed_forward.shared_expert.down_proj", + "language_model.model.layers.40.self_attn.qkv_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.40.self_attn.attn.impl.softmax", + "language_model.model.layers.40.self_attn.attn.impl.matmul_av", + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.40.self_attn.attn.impl.k_cache", + "language_model.model.layers.40.self_attn.attn.impl.v_cache", + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.40.feed_forward.gate_up_proj", + "language_model.model.layers.40.feed_forward.down_proj", + "language_model.model.layers.41.self_attn.qkv_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.41.self_attn.attn.impl.softmax", + "language_model.model.layers.41.self_attn.attn.impl.matmul_av", + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.41.self_attn.attn.impl.k_cache", + "language_model.model.layers.41.self_attn.attn.impl.v_cache", + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.41.feed_forward.router", + "language_model.model.layers.41.feed_forward.experts", + "language_model.model.layers.41.feed_forward.experts.moe_op", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.41.feed_forward.shared_expert.down_proj", + "language_model.model.layers.42.self_attn.qkv_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.42.self_attn.attn.impl.softmax", + "language_model.model.layers.42.self_attn.attn.impl.matmul_av", + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.42.self_attn.attn.impl.k_cache", + "language_model.model.layers.42.self_attn.attn.impl.v_cache", + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.42.feed_forward.gate_up_proj", + "language_model.model.layers.42.feed_forward.down_proj", + "language_model.model.layers.43.self_attn.qkv_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.43.self_attn.attn.impl.softmax", + "language_model.model.layers.43.self_attn.attn.impl.matmul_av", + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.43.self_attn.attn.impl.k_cache", + "language_model.model.layers.43.self_attn.attn.impl.v_cache", + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.43.feed_forward.router", + "language_model.model.layers.43.feed_forward.experts", + "language_model.model.layers.43.feed_forward.experts.moe_op", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.43.feed_forward.shared_expert.down_proj", + "language_model.model.layers.44.self_attn.qkv_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.44.self_attn.attn.impl.softmax", + "language_model.model.layers.44.self_attn.attn.impl.matmul_av", + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.44.self_attn.attn.impl.k_cache", + "language_model.model.layers.44.self_attn.attn.impl.v_cache", + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.44.feed_forward.gate_up_proj", + "language_model.model.layers.44.feed_forward.down_proj", + "language_model.model.layers.45.self_attn.qkv_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.45.self_attn.attn.impl.softmax", + "language_model.model.layers.45.self_attn.attn.impl.matmul_av", + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.45.self_attn.attn.impl.k_cache", + "language_model.model.layers.45.self_attn.attn.impl.v_cache", + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.45.feed_forward.router", + "language_model.model.layers.45.feed_forward.experts", + "language_model.model.layers.45.feed_forward.experts.moe_op", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.45.feed_forward.shared_expert.down_proj", + "language_model.model.layers.46.self_attn.qkv_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.46.self_attn.attn.impl.softmax", + "language_model.model.layers.46.self_attn.attn.impl.matmul_av", + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.46.self_attn.attn.impl.k_cache", + "language_model.model.layers.46.self_attn.attn.impl.v_cache", + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.46.feed_forward.gate_up_proj", + "language_model.model.layers.46.feed_forward.down_proj", + "language_model.model.layers.47.self_attn.qkv_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.47.self_attn.attn.impl.softmax", + "language_model.model.layers.47.self_attn.attn.impl.matmul_av", + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.47.self_attn.attn.impl.k_cache", + "language_model.model.layers.47.self_attn.attn.impl.v_cache", + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.47.feed_forward.router", + "language_model.model.layers.47.feed_forward.experts", + "language_model.model.layers.47.feed_forward.experts.moe_op", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.47.feed_forward.shared_expert.down_proj", + "language_model.lm_head" +] \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_2_8.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_2_8.json new file mode 100644 index 000000000000..2606a34f8f85 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_2_8.json @@ -0,0 +1,24263 @@ +{ + "GlobalRank": null, + "LocalRank": 2, + "Mode": "DynamicRange", + "Nodes": { + "language_model.model.layers.0.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 11.1875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3203125 + ] + ] + } + }, + "language_model.model.layers.0.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.81640625 + ] + ] + ], + "outputs": [ + [ + [ + 0.83203125 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.50390625 + ] + ], + [ + [ + 5.625 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.81640625 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.50390625 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.6328125 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 5.625 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.81640625 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 6.28125 + ] + ], + [ + [ + 5.625 + ] + ], + [ + [ + 0.81640625 + ] + ] + ], + "outputs": [ + [ + [ + 0.81640625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.0.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 2.546875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40234375 + ] + ] + } + }, + "language_model.model.layers.0.feed_forward.down_proj": { + "inputs": [ + [ + [ + 4.25 + ] + ] + ], + "outputs": [ + [ + [ + 26.75 + ] + ], + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.75390625 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 10.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.94140625 + ] + ] + ], + "outputs": [ + [ + [ + 0.67578125 + ] + ], + [ + [ + 1.3984375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.67578125 + ] + ], + [ + [ + 7.0 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2421875 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.67578125 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.875 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 7.0 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.2421875 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 8.4375 + ] + ], + [ + [ + 7.0 + ] + ], + [ + [ + 1.234375 + ] + ] + ], + "outputs": [ + [ + [ + 0.94140625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.1.feed_forward.router": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "outputs": [ + [ + [ + 1.8984375 + ] + ], + [ + [ + 1.0390625 + ] + ], + [ + [ + 10.0625 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 21.125 + ] + ], + [ + [ + 14.8125 + ] + ], + [ + [ + 18.375 + ] + ], + [ + [ + 12.4375 + ] + ], + [ + [ + 22.25 + ] + ], + [ + [ + 16.125 + ] + ], + [ + [ + 24.75 + ] + ], + [ + [ + 11.875 + ] + ], + [ + [ + 17.25 + ] + ], + [ + [ + 7.59375 + ] + ], + [ + [ + 19.125 + ] + ], + [ + [ + 13.125 + ] + ], + [ + [ + 17.25 + ] + ] + ] + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11181640625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.349609375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 5.78125 + ] + ] + ], + "outputs": [ + [ + [ + 1.125 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 9.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.7578125 + ] + ] + ], + "outputs": [ + [ + [ + 0.7734375 + ] + ], + [ + [ + 1.4921875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.625 + ] + ], + [ + [ + 13.375 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.390625 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.625 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.68359375 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.375 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.390625 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 8.0 + ] + ], + [ + [ + 13.375 + ] + ], + [ + [ + 1.390625 + ] + ] + ], + "outputs": [ + [ + [ + 0.7578125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.2.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 6.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.2.feed_forward.down_proj": { + "inputs": [ + [ + [ + 5.46875 + ] + ] + ], + "outputs": [ + [ + [ + 4.75 + ] + ], + [ + [ + 25.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.51953125 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 7.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.458984375 + ] + ] + ], + "outputs": [ + [ + [ + 0.77734375 + ] + ], + [ + [ + 2.40625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.322265625 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.1767578125 + ] + ], + [ + [ + 2.4375 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.35546875 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.1767578125 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.35546875 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 2.578125 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.455078125 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 1.953125 + ] + ], + [ + [ + 2.625 + ] + ], + [ + [ + 0.46484375 + ] + ] + ], + "outputs": [ + [ + [ + 0.458984375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.3.feed_forward.router": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.52734375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "outputs": [ + [ + [ + 2.0625 + ] + ], + [ + [ + 19.125 + ] + ], + [ + [ + 17.875 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 18.875 + ] + ], + [ + [ + 8.375 + ] + ], + [ + [ + 16.125 + ] + ], + [ + [ + 19.25 + ] + ], + [ + [ + 12.0625 + ] + ], + [ + [ + 17.625 + ] + ], + [ + [ + 20.375 + ] + ], + [ + [ + 33.25 + ] + ], + [ + [ + 26.25 + ] + ], + [ + [ + 24.375 + ] + ], + [ + [ + 15.75 + ] + ], + [ + [ + 23.5 + ] + ], + [ + [ + 17.75 + ] + ] + ] + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08935546875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1005859375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08642578125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1044921875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11376953125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 17.375 + ] + ] + ], + "outputs": [ + [ + [ + 1.6875 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 14.3125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.7578125 + ] + ] + ], + "outputs": [ + [ + [ + 1.46875 + ] + ], + [ + [ + 2.328125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.9765625 + ] + ], + [ + [ + 15.375 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.109375 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.9765625 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.140625 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.375 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.109375 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.5625 + ] + ], + [ + [ + 15.375 + ] + ], + [ + [ + 3.109375 + ] + ] + ], + "outputs": [ + [ + [ + 1.7578125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.4.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 6.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.4.feed_forward.down_proj": { + "inputs": [ + [ + [ + 6.59375 + ] + ] + ], + "outputs": [ + [ + [ + 2.796875 + ] + ], + [ + [ + 5.1875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5859375 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 18.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.421875 + ] + ] + ], + "outputs": [ + [ + [ + 1.4921875 + ] + ], + [ + [ + 2.40625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.87109375 + ] + ], + [ + [ + 13.875 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.328125 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.87109375 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.296875 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.875 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.328125 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.6875 + ] + ], + [ + [ + 13.8125 + ] + ], + [ + [ + 2.328125 + ] + ] + ], + "outputs": [ + [ + [ + 1.421875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.5.feed_forward.router": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 2.40625 + ] + ], + [ + [ + 32.0 + ] + ], + [ + [ + 24.875 + ] + ], + [ + [ + 33.5 + ] + ], + [ + [ + 33.5 + ] + ], + [ + [ + 37.25 + ] + ], + [ + [ + 22.75 + ] + ], + [ + [ + 11.375 + ] + ], + [ + [ + 29.75 + ] + ], + [ + [ + 26.25 + ] + ], + [ + [ + 16.875 + ] + ], + [ + [ + 27.25 + ] + ], + [ + [ + 26.375 + ] + ], + [ + [ + 18.75 + ] + ], + [ + [ + 20.25 + ] + ], + [ + [ + 15.8125 + ] + ], + [ + [ + 24.125 + ] + ] + ] + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08984375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1083984375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11572265625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11279296875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08203125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6640625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11767578125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.59375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 7.4375 + ] + ] + ], + "outputs": [ + [ + [ + 1.1328125 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 14.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.453125 + ] + ] + ], + "outputs": [ + [ + [ + 0.94140625 + ] + ], + [ + [ + 1.96875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.65625 + ] + ], + [ + [ + 9.25 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.296875 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.65625 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.046875 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 9.25 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.296875 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.71875 + ] + ], + [ + [ + 9.25 + ] + ], + [ + [ + 3.296875 + ] + ] + ], + "outputs": [ + [ + [ + 2.453125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.6.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 7.71875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.244140625 + ] + ] + } + }, + "language_model.model.layers.6.feed_forward.down_proj": { + "inputs": [ + [ + [ + 7.5 + ] + ] + ], + "outputs": [ + [ + [ + 2.25 + ] + ], + [ + [ + 11.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.546875 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 7.59375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.1015625 + ] + ] + ], + "outputs": [ + [ + [ + 2.1875 + ] + ], + [ + [ + 2.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.482421875 + ] + ], + [ + [ + 5.1875 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.8125 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.482421875 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2265625 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 5.1875 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.8125 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 5.4375 + ] + ], + [ + [ + 5.1875 + ] + ], + [ + [ + 1.8125 + ] + ] + ], + "outputs": [ + [ + [ + 1.1015625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.7.feed_forward.router": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "outputs": [ + [ + [ + 4.0625 + ] + ], + [ + [ + 53.75 + ] + ], + [ + [ + 26.625 + ] + ], + [ + [ + 10.9375 + ] + ], + [ + [ + 17.5 + ] + ], + [ + [ + 28.875 + ] + ], + [ + [ + 41.5 + ] + ], + [ + [ + 58.25 + ] + ], + [ + [ + 29.0 + ] + ], + [ + [ + 27.0 + ] + ], + [ + [ + 15.1875 + ] + ], + [ + [ + 10.4375 + ] + ], + [ + [ + 42.0 + ] + ], + [ + [ + 21.0 + ] + ], + [ + [ + 41.75 + ] + ], + [ + [ + 16.875 + ] + ], + [ + [ + 32.25 + ] + ] + ] + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08642578125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0869140625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0859375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5859375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08544921875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 8.125 + ] + ] + ], + "outputs": [ + [ + [ + 0.7578125 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.390625 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.03125 + ] + ] + ], + "outputs": [ + [ + [ + 2.265625 + ] + ], + [ + [ + 3.390625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.71875 + ] + ], + [ + [ + 8.875 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.734375 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.71875 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.09375 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 8.875 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.734375 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.625 + ] + ], + [ + [ + 8.875 + ] + ], + [ + [ + 3.734375 + ] + ] + ], + "outputs": [ + [ + [ + 2.03125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.8.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 8.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.8.feed_forward.down_proj": { + "inputs": [ + [ + [ + 10.0625 + ] + ] + ], + "outputs": [ + [ + [ + 2.84375 + ] + ], + [ + [ + 9.8125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.349609375 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 16.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.5625 + ] + ] + ], + "outputs": [ + [ + [ + 1.609375 + ] + ], + [ + [ + 3.640625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.75 + ] + ], + [ + [ + 15.3125 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.578125 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.75 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.921875 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.3125 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.578125 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.25 + ] + ], + [ + [ + 15.3125 + ] + ], + [ + [ + 3.578125 + ] + ] + ], + "outputs": [ + [ + [ + 1.5625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.9.feed_forward.router": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "outputs": [ + [ + [ + 11.5 + ] + ], + [ + [ + 45.0 + ] + ], + [ + [ + 45.25 + ] + ], + [ + [ + 56.25 + ] + ], + [ + [ + 19.25 + ] + ], + [ + [ + 57.75 + ] + ], + [ + [ + 32.5 + ] + ], + [ + [ + 37.5 + ] + ], + [ + [ + 37.0 + ] + ], + [ + [ + 44.25 + ] + ], + [ + [ + 16.25 + ] + ], + [ + [ + 30.0 + ] + ], + [ + [ + 47.0 + ] + ], + [ + [ + 40.0 + ] + ], + [ + [ + 37.0 + ] + ], + [ + [ + 31.875 + ] + ], + [ + [ + 53.0 + ] + ] + ] + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.39453125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11376953125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11279296875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08984375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.63671875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 12.5 + ] + ] + ], + "outputs": [ + [ + [ + 1.921875 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.357421875 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 17.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.171875 + ] + ] + ], + "outputs": [ + [ + [ + 1.984375 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.91796875 + ] + ], + [ + [ + 17.875 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.796875 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.91796875 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.8984375 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.875 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.796875 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.4375 + ] + ], + [ + [ + 17.875 + ] + ], + [ + [ + 2.796875 + ] + ] + ], + "outputs": [ + [ + [ + 2.171875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.10.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 9.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.10.feed_forward.down_proj": { + "inputs": [ + [ + [ + 11.5625 + ] + ] + ], + "outputs": [ + [ + [ + 3.40625 + ] + ], + [ + [ + 12.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.57421875 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 15.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.953125 + ] + ] + ], + "outputs": [ + [ + [ + 8.625 + ] + ], + [ + [ + 8.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.921875 + ] + ], + [ + [ + 10.9375 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.53125 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.921875 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.5 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 10.9375 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.53125 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.375 + ] + ], + [ + [ + 10.9375 + ] + ], + [ + [ + 4.53125 + ] + ] + ], + "outputs": [ + [ + [ + 2.953125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.11.feed_forward.router": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "outputs": [ + [ + [ + 4.59375 + ] + ], + [ + [ + 33.5 + ] + ], + [ + [ + 30.625 + ] + ], + [ + [ + 52.5 + ] + ], + [ + [ + 36.75 + ] + ], + [ + [ + 28.75 + ] + ], + [ + [ + 41.75 + ] + ], + [ + [ + 51.25 + ] + ], + [ + [ + 51.5 + ] + ], + [ + [ + 18.75 + ] + ], + [ + [ + 50.0 + ] + ], + [ + [ + 40.75 + ] + ], + [ + [ + 50.5 + ] + ], + [ + [ + 47.25 + ] + ], + [ + [ + 49.25 + ] + ], + [ + [ + 44.0 + ] + ], + [ + [ + 61.25 + ] + ] + ] + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11279296875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11572265625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1044921875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1005859375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09326171875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08154296875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08447265625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.091796875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.103515625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 13.5625 + ] + ] + ], + "outputs": [ + [ + [ + 3.46875 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.56640625 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 20.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.953125 + ] + ] + ], + "outputs": [ + [ + [ + 1.796875 + ] + ], + [ + [ + 3.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1484375 + ] + ], + [ + [ + 12.0 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1484375 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.953125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 12.0 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.0 + ] + ], + [ + [ + 12.0 + ] + ], + [ + [ + 3.125 + ] + ] + ], + "outputs": [ + [ + [ + 2.953125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.12.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 10.6875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.12.feed_forward.down_proj": { + "inputs": [ + [ + [ + 15.3125 + ] + ] + ], + "outputs": [ + [ + [ + 9.25 + ] + ], + [ + [ + 20.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37109375 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.578125 + ] + ] + ], + "outputs": [ + [ + [ + 2.015625 + ] + ], + [ + [ + 6.34375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.447265625 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.828125 + ] + ], + [ + [ + 16.875 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.109375 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.828125 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.140625 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.875 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.109375 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.3125 + ] + ], + [ + [ + 16.5 + ] + ], + [ + [ + 3.109375 + ] + ] + ], + "outputs": [ + [ + [ + 2.578125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.13.feed_forward.router": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "outputs": [ + [ + [ + 7.25 + ] + ], + [ + [ + 59.0 + ] + ], + [ + [ + 55.25 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 59.0 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 60.75 + ] + ], + [ + [ + 35.0 + ] + ], + [ + [ + 22.875 + ] + ], + [ + [ + 61.25 + ] + ], + [ + [ + 62.75 + ] + ], + [ + [ + 50.0 + ] + ], + [ + [ + 25.125 + ] + ], + [ + [ + 41.5 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 40.25 + ] + ], + [ + [ + 25.5 + ] + ] + ] + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10498046875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1083984375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5546875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08642578125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0859375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3046875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 14.3125 + ] + ] + ], + "outputs": [ + [ + [ + 10.625 + ] + ], + [ + [ + 1.906546502743257e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.74609375 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 20.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2353515625 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.390625 + ] + ] + ], + "outputs": [ + [ + [ + 3.546875 + ] + ], + [ + [ + 5.71875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.828125 + ] + ], + [ + [ + 13.0625 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.125 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.828125 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.921875 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.0625 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.125 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.9375 + ] + ], + [ + [ + 13.0625 + ] + ], + [ + [ + 3.78125 + ] + ] + ], + "outputs": [ + [ + [ + 3.390625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.14.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 12.0625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.14.feed_forward.down_proj": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ], + "outputs": [ + [ + [ + 5.375 + ] + ], + [ + [ + 20.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.78515625 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 23.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.890625 + ] + ] + ], + "outputs": [ + [ + [ + 20.0 + ] + ], + [ + [ + 21.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.91015625 + ] + ], + [ + [ + 11.25 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.15625 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.91015625 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.625 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 11.25 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.15625 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.5 + ] + ], + [ + [ + 11.25 + ] + ], + [ + [ + 5.15625 + ] + ] + ], + "outputs": [ + [ + [ + 3.890625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.15.feed_forward.router": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 6.59375 + ] + ], + [ + [ + 42.5 + ] + ], + [ + [ + 41.0 + ] + ], + [ + [ + 38.0 + ] + ], + [ + [ + 52.25 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 45.75 + ] + ], + [ + [ + 66.5 + ] + ], + [ + [ + 48.25 + ] + ], + [ + [ + 60.0 + ] + ], + [ + [ + 62.0 + ] + ], + [ + [ + 44.0 + ] + ], + [ + [ + 81.0 + ] + ], + [ + [ + 56.0 + ] + ], + [ + [ + 66.5 + ] + ], + [ + [ + 67.5 + ] + ], + [ + [ + 85.5 + ] + ] + ] + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2353515625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09130859375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1083984375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08984375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09130859375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 14.375 + ] + ] + ], + "outputs": [ + [ + [ + 3.4375 + ] + ], + [ + [ + 2.5799225015844925e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.357421875 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 23.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.21875 + ] + ] + ], + "outputs": [ + [ + [ + 2.4375 + ] + ], + [ + [ + 6.3125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0859375 + ] + ], + [ + [ + 13.4375 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.75 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0859375 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.8125 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.4375 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.75 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.125 + ] + ], + [ + [ + 13.4375 + ] + ], + [ + [ + 6.75 + ] + ] + ], + "outputs": [ + [ + [ + 6.21875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.16.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 14.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.16.feed_forward.down_proj": { + "inputs": [ + [ + [ + 17.25 + ] + ] + ], + "outputs": [ + [ + [ + 6.65625 + ] + ], + [ + [ + 27.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.53125 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.5 + ] + ] + ], + "outputs": [ + [ + [ + 19.5 + ] + ], + [ + [ + 27.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.330078125 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.91796875 + ] + ], + [ + [ + 10.875 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.65625 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.91796875 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.71875 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 10.875 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.65625 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.4375 + ] + ], + [ + [ + 10.6875 + ] + ], + [ + [ + 5.65625 + ] + ] + ], + "outputs": [ + [ + [ + 5.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.17.feed_forward.router": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 8.25 + ] + ], + [ + [ + 60.75 + ] + ], + [ + [ + 49.5 + ] + ], + [ + [ + 68.0 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 99.5 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 89.0 + ] + ], + [ + [ + 34.75 + ] + ], + [ + [ + 72.5 + ] + ], + [ + [ + 66.5 + ] + ], + [ + [ + 80.0 + ] + ], + [ + [ + 56.75 + ] + ], + [ + [ + 60.25 + ] + ], + [ + [ + 74.0 + ] + ], + [ + [ + 72.0 + ] + ] + ] + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2353515625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.283203125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.365234375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 27.75 + ] + ] + ], + "outputs": [ + [ + [ + 25.75 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.92578125 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.171875 + ] + ] + ], + "outputs": [ + [ + [ + 6.625 + ] + ], + [ + [ + 12.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.125 + ] + ], + [ + [ + 18.375 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.75 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.125 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.09375 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.375 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.75 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.375 + ] + ], + [ + [ + 18.375 + ] + ], + [ + [ + 3.75 + ] + ] + ], + "outputs": [ + [ + [ + 3.171875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.18.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 20.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.18.feed_forward.down_proj": { + "inputs": [ + [ + [ + 264.0 + ] + ] + ], + "outputs": [ + [ + [ + 8.1875 + ] + ], + [ + [ + 354.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.70703125 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 44.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.96875 + ] + ] + ], + "outputs": [ + [ + [ + 31.5 + ] + ], + [ + [ + 37.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.8828125 + ] + ], + [ + [ + 11.0625 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.8828125 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.96875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 11.0625 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.3125 + ] + ], + [ + [ + 11.0625 + ] + ], + [ + [ + 5.875 + ] + ] + ], + "outputs": [ + [ + [ + 3.8125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.19.feed_forward.router": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "outputs": [ + [ + [ + 8.625 + ] + ], + [ + [ + 54.5 + ] + ], + [ + [ + 61.5 + ] + ], + [ + [ + 58.5 + ] + ], + [ + [ + 62.25 + ] + ], + [ + [ + 48.75 + ] + ], + [ + [ + 52.5 + ] + ], + [ + [ + 92.0 + ] + ], + [ + [ + 77.5 + ] + ], + [ + [ + 72.5 + ] + ], + [ + [ + 50.5 + ] + ], + [ + [ + 57.25 + ] + ], + [ + [ + 75.0 + ] + ], + [ + [ + 52.75 + ] + ], + [ + [ + 39.75 + ] + ], + [ + [ + 35.25 + ] + ], + [ + [ + 81.5 + ] + ] + ] + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5546875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3984375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.50390625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "outputs": [ + [ + [ + 3.609375 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 53.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.5 + ] + ] + ], + "outputs": [ + [ + [ + 12.8125 + ] + ], + [ + [ + 22.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.3515625 + ] + ], + [ + [ + 17.875 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.0 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.3515625 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.125 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.875 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.0 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.875 + ] + ], + [ + [ + 17.875 + ] + ], + [ + [ + 7.0 + ] + ] + ], + "outputs": [ + [ + [ + 4.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.20.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 42.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.32421875 + ] + ] + } + }, + "language_model.model.layers.20.feed_forward.down_proj": { + "inputs": [ + [ + [ + 19.625 + ] + ] + ], + "outputs": [ + [ + [ + 10.5 + ] + ], + [ + [ + 41.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.84375 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 58.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.6875 + ] + ] + ], + "outputs": [ + [ + [ + 26.0 + ] + ], + [ + [ + 35.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6484375 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.3046875 + ] + ], + [ + [ + 13.375 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.09375 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.3046875 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.375 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.375 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.09375 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.125 + ] + ], + [ + [ + 13.375 + ] + ], + [ + [ + 7.09375 + ] + ] + ], + "outputs": [ + [ + [ + 4.6875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.21.feed_forward.router": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.45703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "outputs": [ + [ + [ + 8.3125 + ] + ], + [ + [ + 64.5 + ] + ], + [ + [ + 49.5 + ] + ], + [ + [ + 36.75 + ] + ], + [ + [ + 50.25 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 104.0 + ] + ], + [ + [ + 86.5 + ] + ], + [ + [ + 88.0 + ] + ], + [ + [ + 53.0 + ] + ], + [ + [ + 53.5 + ] + ], + [ + [ + 48.0 + ] + ], + [ + [ + 79.5 + ] + ], + [ + [ + 58.75 + ] + ], + [ + [ + 94.5 + ] + ], + [ + [ + 84.0 + ] + ], + [ + [ + 91.0 + ] + ] + ] + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.462890625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.244140625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.396484375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.392578125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 25.75 + ] + ] + ], + "outputs": [ + [ + [ + 4.65625 + ] + ], + [ + [ + 2.596148429267414e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.50390625 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 57.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.59375 + ] + ] + ], + "outputs": [ + [ + [ + 15.75 + ] + ], + [ + [ + 28.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0546875 + ] + ], + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.78125 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0546875 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.25 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.78125 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.25 + ] + ], + [ + [ + 17.375 + ] + ], + [ + [ + 7.78125 + ] + ] + ], + "outputs": [ + [ + [ + 5.59375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.22.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 46.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.50390625 + ] + ] + } + }, + "language_model.model.layers.22.feed_forward.down_proj": { + "inputs": [ + [ + [ + 44.25 + ] + ] + ], + "outputs": [ + [ + [ + 14.875 + ] + ], + [ + [ + 77.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.74609375 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 69.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.5 + ] + ] + ], + "outputs": [ + [ + [ + 70.0 + ] + ], + [ + [ + 79.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.80859375 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.921875 + ] + ], + [ + [ + 18.125 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.6875 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.921875 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.875 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.125 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.6875 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.4375 + ] + ], + [ + [ + 18.125 + ] + ], + [ + [ + 8.6875 + ] + ] + ], + "outputs": [ + [ + [ + 5.125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.23.feed_forward.router": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.474609375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 33.0 + ] + ], + [ + [ + 72.0 + ] + ], + [ + [ + 32.25 + ] + ], + [ + [ + 105.0 + ] + ], + [ + [ + 53.25 + ] + ], + [ + [ + 104.0 + ] + ], + [ + [ + 51.25 + ] + ], + [ + [ + 100.5 + ] + ], + [ + [ + 52.75 + ] + ], + [ + [ + 70.5 + ] + ], + [ + [ + 43.5 + ] + ], + [ + [ + 55.25 + ] + ], + [ + [ + 46.5 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 102.5 + ] + ], + [ + [ + 85.5 + ] + ], + [ + [ + 119.0 + ] + ] + ] + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10498046875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.283203125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3984375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34765625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 21.25 + ] + ] + ], + "outputs": [ + [ + [ + 9.25 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.490234375 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.53125 + ] + ] + ], + "outputs": [ + [ + [ + 48.0 + ] + ], + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.125 + ] + ], + [ + [ + 19.0 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.5625 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.125 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.25 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.0 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.5625 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.75 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 11.5625 + ] + ] + ], + "outputs": [ + [ + [ + 6.03125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.24.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.494140625 + ] + ] + } + }, + "language_model.model.layers.24.feed_forward.down_proj": { + "inputs": [ + [ + [ + 38.5 + ] + ] + ], + "outputs": [ + [ + [ + 21.75 + ] + ], + [ + [ + 100.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5546875 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.1875 + ] + ] + ], + "outputs": [ + [ + [ + 53.0 + ] + ], + [ + [ + 69.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.39453125 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1484375 + ] + ], + [ + [ + 11.5625 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.625 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1484375 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.0625 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 11.5625 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 12.625 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.9375 + ] + ], + [ + [ + 11.5625 + ] + ], + [ + [ + 12.625 + ] + ] + ], + "outputs": [ + [ + [ + 6.1875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.25.feed_forward.router": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.412109375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "outputs": [ + [ + [ + 11.4375 + ] + ], + [ + [ + 69.0 + ] + ], + [ + [ + 97.5 + ] + ], + [ + [ + 51.5 + ] + ], + [ + [ + 57.75 + ] + ], + [ + [ + 67.0 + ] + ], + [ + [ + 69.0 + ] + ], + [ + [ + 56.75 + ] + ], + [ + [ + 86.0 + ] + ], + [ + [ + 34.25 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 40.75 + ] + ], + [ + [ + 94.5 + ] + ], + [ + [ + 65.5 + ] + ], + [ + [ + 73.5 + ] + ], + [ + [ + 150.0 + ] + ], + [ + [ + 110.5 + ] + ] + ] + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23828125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3671875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 50.75 + ] + ] + ], + "outputs": [ + [ + [ + 43.0 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.76953125 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 63.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.46875 + ] + ] + ], + "outputs": [ + [ + [ + 30.875 + ] + ], + [ + [ + 48.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6796875 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1171875 + ] + ], + [ + [ + 16.25 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.90625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1171875 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.0625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.25 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.90625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.6875 + ] + ], + [ + [ + 15.9375 + ] + ], + [ + [ + 6.90625 + ] + ] + ], + "outputs": [ + [ + [ + 4.46875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.26.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 45.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.451171875 + ] + ] + } + }, + "language_model.model.layers.26.feed_forward.down_proj": { + "inputs": [ + [ + [ + 24.25 + ] + ] + ], + "outputs": [ + [ + [ + 22.125 + ] + ], + [ + [ + 113.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.53515625 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.71875 + ] + ] + ], + "outputs": [ + [ + [ + 60.25 + ] + ], + [ + [ + 71.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.3046875 + ] + ], + [ + [ + 16.75 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.25 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.3046875 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.09375 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.75 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.25 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.25 + ] + ], + [ + [ + 16.75 + ] + ], + [ + [ + 9.25 + ] + ] + ], + "outputs": [ + [ + [ + 6.71875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.27.feed_forward.router": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "outputs": [ + [ + [ + 10.125 + ] + ], + [ + [ + 94.0 + ] + ], + [ + [ + 39.5 + ] + ], + [ + [ + 90.5 + ] + ], + [ + [ + 41.25 + ] + ], + [ + [ + 62.75 + ] + ], + [ + [ + 95.5 + ] + ], + [ + [ + 72.5 + ] + ], + [ + [ + 52.5 + ] + ], + [ + [ + 45.75 + ] + ], + [ + [ + 50.5 + ] + ], + [ + [ + 93.5 + ] + ], + [ + [ + 44.75 + ] + ], + [ + [ + 53.0 + ] + ], + [ + [ + 53.0 + ] + ], + [ + [ + 86.0 + ] + ], + [ + [ + 50.75 + ] + ] + ] + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.341796875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.412109375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 40.75 + ] + ] + ], + "outputs": [ + [ + [ + 53.5 + ] + ], + [ + [ + 2.5799225015844925e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 1.03125 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 78.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.09375 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 97.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0859375 + ] + ], + [ + [ + 17.0 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.0625 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0859375 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.84375 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.0 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.0625 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.375 + ] + ], + [ + [ + 16.875 + ] + ], + [ + [ + 10.0625 + ] + ] + ], + "outputs": [ + [ + [ + 5.09375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.28.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.62109375 + ] + ] + } + }, + "language_model.model.layers.28.feed_forward.down_proj": { + "inputs": [ + [ + [ + 34.5 + ] + ] + ], + "outputs": [ + [ + [ + 140.0 + ] + ], + [ + [ + 149.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.828125 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 67.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.o_proj": { + "inputs": [ + [ + [ + 9.0625 + ] + ] + ], + "outputs": [ + [ + [ + 78.0 + ] + ], + [ + [ + 84.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.91796875 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.4921875 + ] + ], + [ + [ + 22.5 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.25 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4921875 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.875 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 22.5 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.25 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 19.75 + ] + ], + [ + [ + 22.5 + ] + ], + [ + [ + 9.25 + ] + ] + ], + "outputs": [ + [ + [ + 9.0625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.29.feed_forward.router": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.333984375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "outputs": [ + [ + [ + 10.9375 + ] + ], + [ + [ + 86.5 + ] + ], + [ + [ + 66.5 + ] + ], + [ + [ + 74.0 + ] + ], + [ + [ + 70.0 + ] + ], + [ + [ + 47.5 + ] + ], + [ + [ + 36.5 + ] + ], + [ + [ + 94.5 + ] + ], + [ + [ + 83.5 + ] + ], + [ + [ + 75.0 + ] + ], + [ + [ + 58.5 + ] + ], + [ + [ + 18.5 + ] + ], + [ + [ + 75.0 + ] + ], + [ + [ + 103.5 + ] + ], + [ + [ + 83.0 + ] + ], + [ + [ + 50.75 + ] + ], + [ + [ + 77.5 + ] + ] + ] + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37109375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2490234375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.365234375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 48.25 + ] + ] + ], + "outputs": [ + [ + [ + 19.875 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8046875 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.0625 + ] + ] + ], + "outputs": [ + [ + [ + 50.25 + ] + ], + [ + [ + 56.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.421875 + ] + ], + [ + [ + 21.75 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.375 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.421875 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.6875 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 21.75 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.375 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 19.375 + ] + ], + [ + [ + 21.75 + ] + ], + [ + [ + 10.375 + ] + ] + ], + "outputs": [ + [ + [ + 7.0625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.30.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 32.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.484375 + ] + ] + } + }, + "language_model.model.layers.30.feed_forward.down_proj": { + "inputs": [ + [ + [ + 33.25 + ] + ] + ], + "outputs": [ + [ + [ + 30.25 + ] + ], + [ + [ + 159.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.609375 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.875 + ] + ] + ], + "outputs": [ + [ + [ + 77.0 + ] + ], + [ + [ + 115.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.09375 + ] + ], + [ + [ + 13.4375 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 17.125 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.09375 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.8125 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.4375 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.6875 + ] + ], + [ + [ + 13.4375 + ] + ], + [ + [ + 17.125 + ] + ] + ], + "outputs": [ + [ + [ + 8.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.31.feed_forward.router": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "outputs": [ + [ + [ + 35.75 + ] + ], + [ + [ + 95.5 + ] + ], + [ + [ + 76.0 + ] + ], + [ + [ + 86.5 + ] + ], + [ + [ + 45.75 + ] + ], + [ + [ + 59.5 + ] + ], + [ + [ + 48.75 + ] + ], + [ + [ + 125.5 + ] + ], + [ + [ + 84.5 + ] + ], + [ + [ + 83.0 + ] + ], + [ + [ + 97.5 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 54.25 + ] + ], + [ + [ + 103.5 + ] + ], + [ + [ + 91.5 + ] + ], + [ + [ + 56.5 + ] + ], + [ + [ + 87.0 + ] + ] + ] + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33203125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33203125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.421875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.380859375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2353515625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.330078125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5234375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 59.25 + ] + ] + ], + "outputs": [ + [ + [ + 58.0 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.99609375 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 53.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.21875 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 109.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.25 + ] + ], + [ + [ + 18.5 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.21875 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.25 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.375 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.21875 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.3125 + ] + ], + [ + [ + 18.5 + ] + ], + [ + [ + 6.21875 + ] + ] + ], + "outputs": [ + [ + [ + 5.21875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.32.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 32.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.287109375 + ] + ] + } + }, + "language_model.model.layers.32.feed_forward.down_proj": { + "inputs": [ + [ + [ + 56.25 + ] + ] + ], + "outputs": [ + [ + [ + 29.375 + ] + ], + [ + [ + 75.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.490234375 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 44.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.5625 + ] + ] + ], + "outputs": [ + [ + [ + 94.0 + ] + ], + [ + [ + 132.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.140625 + ] + ], + [ + [ + 22.375 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.65625 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.140625 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.84375 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 22.375 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.65625 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.5625 + ] + ], + [ + [ + 22.375 + ] + ], + [ + [ + 6.65625 + ] + ] + ], + "outputs": [ + [ + [ + 5.5625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.33.feed_forward.router": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "outputs": [ + [ + [ + 25.0 + ] + ], + [ + [ + 34.75 + ] + ], + [ + [ + 113.0 + ] + ], + [ + [ + 103.0 + ] + ], + [ + [ + 103.0 + ] + ], + [ + [ + 86.5 + ] + ], + [ + [ + 30.875 + ] + ], + [ + [ + 99.0 + ] + ], + [ + [ + 46.5 + ] + ], + [ + [ + 92.0 + ] + ], + [ + [ + 59.0 + ] + ], + [ + [ + 91.5 + ] + ], + [ + [ + 87.0 + ] + ], + [ + [ + 60.75 + ] + ], + [ + [ + 25.25 + ] + ], + [ + [ + 129.0 + ] + ], + [ + [ + 115.0 + ] + ] + ] + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1025390625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11181640625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10498046875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.333984375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08544921875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 67.5 + ] + ] + ], + "outputs": [ + [ + [ + 13.125 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.435546875 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 54.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.96875 + ] + ] + ], + "outputs": [ + [ + [ + 55.75 + ] + ], + [ + [ + 98.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2890625 + ] + ], + [ + [ + 21.625 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.3125 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2890625 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.96875 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 21.625 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.3125 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.4375 + ] + ], + [ + [ + 21.625 + ] + ], + [ + [ + 8.0625 + ] + ] + ], + "outputs": [ + [ + [ + 7.28125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.34.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.376953125 + ] + ] + } + }, + "language_model.model.layers.34.feed_forward.down_proj": { + "inputs": [ + [ + [ + 66.5 + ] + ] + ], + "outputs": [ + [ + [ + 36.5 + ] + ], + [ + [ + 162.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.546875 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 54.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.o_proj": { + "inputs": [ + [ + [ + 10.0 + ] + ] + ], + "outputs": [ + [ + [ + 110.5 + ] + ], + [ + [ + 116.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.330078125 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0859375 + ] + ], + [ + [ + 18.75 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.75 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0859375 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.75 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.75 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 12.75 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.0 + ] + ], + [ + [ + 18.75 + ] + ], + [ + [ + 12.75 + ] + ] + ], + "outputs": [ + [ + [ + 10.0 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.35.feed_forward.router": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "outputs": [ + [ + [ + 24.625 + ] + ], + [ + [ + 126.5 + ] + ], + [ + [ + 81.0 + ] + ], + [ + [ + 141.0 + ] + ], + [ + [ + 140.0 + ] + ], + [ + [ + 225.0 + ] + ], + [ + [ + 111.0 + ] + ], + [ + [ + 137.0 + ] + ], + [ + [ + 116.0 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 137.0 + ] + ], + [ + [ + 31.75 + ] + ], + [ + [ + 126.0 + ] + ], + [ + [ + 132.0 + ] + ], + [ + [ + 116.0 + ] + ], + [ + [ + 75.0 + ] + ], + [ + [ + 64.5 + ] + ] + ] + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09130859375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33203125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1044921875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3203125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.53125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11572265625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5078125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 120.5 + ] + ] + ], + "outputs": [ + [ + [ + 43.25 + ] + ], + [ + [ + 2.7259558507307845e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6171875 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 56.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.o_proj": { + "inputs": [ + [ + [ + 10.0 + ] + ] + ], + "outputs": [ + [ + [ + 36.75 + ] + ], + [ + [ + 57.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3984375 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.328125 + ] + ], + [ + [ + 22.125 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.4375 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.328125 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.6875 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 22.125 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.4375 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.25 + ] + ], + [ + [ + 22.125 + ] + ], + [ + [ + 10.4375 + ] + ] + ], + "outputs": [ + [ + [ + 10.0 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.36.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 29.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.36.feed_forward.down_proj": { + "inputs": [ + [ + [ + 69.0 + ] + ] + ], + "outputs": [ + [ + [ + 43.0 + ] + ], + [ + [ + 94.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.9453125 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.o_proj": { + "inputs": [ + [ + [ + 11.4375 + ] + ] + ], + "outputs": [ + [ + [ + 97.5 + ] + ], + [ + [ + 150.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.984375 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.3984375 + ] + ], + [ + [ + 12.0625 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.0 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.3984375 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.125 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 12.0625 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.0 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.0 + ] + ], + [ + [ + 12.0625 + ] + ], + [ + [ + 13.0 + ] + ] + ], + "outputs": [ + [ + [ + 11.4375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.37.feed_forward.router": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "outputs": [ + [ + [ + 39.75 + ] + ], + [ + [ + 55.75 + ] + ], + [ + [ + 142.0 + ] + ], + [ + [ + 217.0 + ] + ], + [ + [ + 78.0 + ] + ], + [ + [ + 116.0 + ] + ], + [ + [ + 181.0 + ] + ], + [ + [ + 167.0 + ] + ], + [ + [ + 123.5 + ] + ], + [ + [ + 110.0 + ] + ], + [ + [ + 60.25 + ] + ], + [ + [ + 101.0 + ] + ], + [ + [ + 99.0 + ] + ], + [ + [ + 118.0 + ] + ], + [ + [ + 151.0 + ] + ], + [ + [ + 145.0 + ] + ], + [ + [ + 117.5 + ] + ] + ] + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.640625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5859375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.359375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.341796875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.283203125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 119.0 + ] + ] + ], + "outputs": [ + [ + [ + 14.875 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.60546875 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 47.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.o_proj": { + "inputs": [ + [ + [ + 13.3125 + ] + ] + ], + "outputs": [ + [ + [ + 29.75 + ] + ], + [ + [ + 66.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.90625 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.78125 + ] + ], + [ + [ + 14.8125 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.4375 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.78125 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.125 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.8125 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.4375 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 21.125 + ] + ], + [ + [ + 14.8125 + ] + ], + [ + [ + 13.4375 + ] + ] + ], + "outputs": [ + [ + [ + 13.3125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.38.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 31.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28515625 + ] + ] + } + }, + "language_model.model.layers.38.feed_forward.down_proj": { + "inputs": [ + [ + [ + 96.5 + ] + ] + ], + "outputs": [ + [ + [ + 31.125 + ] + ], + [ + [ + 262.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6953125 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 52.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.o_proj": { + "inputs": [ + [ + [ + 9.3125 + ] + ] + ], + "outputs": [ + [ + [ + 126.5 + ] + ], + [ + [ + 154.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.3125 + ] + ], + [ + [ + 13.25 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.8125 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.3125 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.8125 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.25 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.8125 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.9375 + ] + ], + [ + [ + 13.25 + ] + ], + [ + [ + 11.8125 + ] + ] + ], + "outputs": [ + [ + [ + 9.3125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.39.feed_forward.router": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "outputs": [ + [ + [ + 35.5 + ] + ], + [ + [ + 364.0 + ] + ], + [ + [ + 109.0 + ] + ], + [ + [ + 189.0 + ] + ], + [ + [ + 115.0 + ] + ], + [ + [ + 126.0 + ] + ], + [ + [ + 216.0 + ] + ], + [ + [ + 118.0 + ] + ], + [ + [ + 124.5 + ] + ], + [ + [ + 198.0 + ] + ], + [ + [ + 102.5 + ] + ], + [ + [ + 141.0 + ] + ], + [ + [ + 116.0 + ] + ], + [ + [ + 98.5 + ] + ], + [ + [ + 126.5 + ] + ], + [ + [ + 98.0 + ] + ], + [ + [ + 101.0 + ] + ] + ] + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37109375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33203125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.474609375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 119.0 + ] + ] + ], + "outputs": [ + [ + [ + 17.875 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4453125 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 46.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.o_proj": { + "inputs": [ + [ + [ + 16.25 + ] + ] + ], + "outputs": [ + [ + [ + 94.0 + ] + ], + [ + [ + 102.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.48828125 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.328125 + ] + ], + [ + [ + 12.875 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 18.25 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.328125 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.125 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 12.875 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 18.25 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.3125 + ] + ], + [ + [ + 12.875 + ] + ], + [ + [ + 18.25 + ] + ] + ], + "outputs": [ + [ + [ + 16.25 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.40.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.32421875 + ] + ] + } + }, + "language_model.model.layers.40.feed_forward.down_proj": { + "inputs": [ + [ + [ + 169.0 + ] + ] + ], + "outputs": [ + [ + [ + 28.5 + ] + ], + [ + [ + 127.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.62109375 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 35.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.o_proj": { + "inputs": [ + [ + [ + 13.125 + ] + ] + ], + "outputs": [ + [ + [ + 160.0 + ] + ], + [ + [ + 177.0 + ] + ] + ], + "params": { + "weight": [ + [ + 1.1875 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.40625 + ] + ], + [ + [ + 20.5 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.1875 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.40625 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.5 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.5 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.1875 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.75 + ] + ], + [ + [ + 20.5 + ] + ], + [ + [ + 13.1875 + ] + ] + ], + "outputs": [ + [ + [ + 13.125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.41.feed_forward.router": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "outputs": [ + [ + [ + 62.5 + ] + ], + [ + [ + 231.0 + ] + ], + [ + [ + 194.0 + ] + ], + [ + [ + 146.0 + ] + ], + [ + [ + 183.0 + ] + ], + [ + [ + 123.0 + ] + ], + [ + [ + 184.0 + ] + ], + [ + [ + 124.5 + ] + ], + [ + [ + 203.0 + ] + ], + [ + [ + 91.5 + ] + ], + [ + [ + 203.0 + ] + ], + [ + [ + 131.0 + ] + ], + [ + [ + 1528.0 + ] + ], + [ + [ + 145.0 + ] + ], + [ + [ + 135.0 + ] + ], + [ + [ + 197.0 + ] + ], + [ + [ + 190.0 + ] + ] + ] + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4453125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4765625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.36328125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 187.0 + ] + ] + ], + "outputs": [ + [ + [ + 32.0 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.65234375 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 39.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.o_proj": { + "inputs": [ + [ + [ + 13.75 + ] + ] + ], + "outputs": [ + [ + [ + 65.0 + ] + ], + [ + [ + 146.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.671875 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.40625 + ] + ], + [ + [ + 17.25 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.875 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.40625 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.375 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.25 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.875 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 17.25 + ] + ], + [ + [ + 17.25 + ] + ], + [ + [ + 13.875 + ] + ] + ], + "outputs": [ + [ + [ + 13.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.42.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 43.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.42.feed_forward.down_proj": { + "inputs": [ + [ + [ + 161.0 + ] + ] + ], + "outputs": [ + [ + [ + 70.5 + ] + ], + [ + [ + 218.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.55859375 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 43.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.o_proj": { + "inputs": [ + [ + [ + 14.125 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 232.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.3515625 + ] + ], + [ + [ + 12.4375 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 27.375 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.3515625 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 15.8125 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 12.4375 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 27.375 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.625 + ] + ], + [ + [ + 12.4375 + ] + ], + [ + [ + 27.25 + ] + ] + ], + "outputs": [ + [ + [ + 14.125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.43.feed_forward.router": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "outputs": [ + [ + [ + 53.0 + ] + ], + [ + [ + 107.5 + ] + ], + [ + [ + 179.0 + ] + ], + [ + [ + 278.0 + ] + ], + [ + [ + 174.0 + ] + ], + [ + [ + 167.0 + ] + ], + [ + [ + 167.0 + ] + ], + [ + [ + 83.0 + ] + ], + [ + [ + 234.0 + ] + ], + [ + [ + 185.0 + ] + ], + [ + [ + 157.0 + ] + ], + [ + [ + 225.0 + ] + ], + [ + [ + 115.5 + ] + ], + [ + [ + 278.0 + ] + ], + [ + [ + 171.0 + ] + ], + [ + [ + 140.0 + ] + ], + [ + [ + 56.25 + ] + ] + ] + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3828125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3046875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.333984375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.359375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5703125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10498046875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.310546875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 222.0 + ] + ] + ], + "outputs": [ + [ + [ + 36.75 + ] + ], + [ + [ + 2.888215127559998e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.625 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 29.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.o_proj": { + "inputs": [ + [ + [ + 27.375 + ] + ] + ], + "outputs": [ + [ + [ + 81.0 + ] + ], + [ + [ + 170.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.4609375 + ] + ], + [ + [ + 12.1875 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 37.5 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4609375 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 21.375 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 12.1875 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 37.5 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.875 + ] + ], + [ + [ + 12.1875 + ] + ], + [ + [ + 37.5 + ] + ] + ], + "outputs": [ + [ + [ + 27.375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.44.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 42.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.44.feed_forward.down_proj": { + "inputs": [ + [ + [ + 198.0 + ] + ] + ], + "outputs": [ + [ + [ + 75.5 + ] + ], + [ + [ + 200.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.63671875 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 28.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.o_proj": { + "inputs": [ + [ + [ + 17.5 + ] + ] + ], + "outputs": [ + [ + [ + 170.0 + ] + ], + [ + [ + 258.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.578125 + ] + ], + [ + [ + 19.875 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 23.875 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.578125 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.0625 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.875 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 23.875 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 21.5 + ] + ], + [ + [ + 19.875 + ] + ], + [ + [ + 23.875 + ] + ] + ], + "outputs": [ + [ + [ + 17.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.45.feed_forward.router": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "outputs": [ + [ + [ + 55.25 + ] + ], + [ + [ + 154.0 + ] + ], + [ + [ + 43.0 + ] + ], + [ + [ + 134.0 + ] + ], + [ + [ + 235.0 + ] + ], + [ + [ + 100.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 108.0 + ] + ], + [ + [ + 23.75 + ] + ], + [ + [ + 115.5 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 146.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 216.0 + ] + ] + ] + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.52734375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10693359375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 232.0 + ] + ] + ], + "outputs": [ + [ + [ + 43.25 + ] + ], + [ + [ + 2.5799225015844925e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5546875 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 27.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.o_proj": { + "inputs": [ + [ + [ + 18.875 + ] + ] + ], + "outputs": [ + [ + [ + 126.0 + ] + ], + [ + [ + 221.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.6484375 + ] + ], + [ + [ + 12.75 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 18.875 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.6484375 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 18.875 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 12.75 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 18.875 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 19.5 + ] + ], + [ + [ + 12.75 + ] + ], + [ + [ + 18.75 + ] + ] + ], + "outputs": [ + [ + [ + 17.625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.46.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 40.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.52734375 + ] + ] + } + }, + "language_model.model.layers.46.feed_forward.down_proj": { + "inputs": [ + [ + [ + 358.0 + ] + ] + ], + "outputs": [ + [ + [ + 1680.0 + ] + ], + [ + [ + 2400.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.59375 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.296875 + ] + ] + ], + "outputs": [ + [ + [ + 96.5 + ] + ], + [ + [ + 125.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3671875 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.70703125 + ] + ], + [ + [ + 5.8125 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.53125 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.70703125 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.03125 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 5.8125 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.53125 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 8.125 + ] + ], + [ + [ + 5.8125 + ] + ], + [ + [ + 5.53125 + ] + ] + ], + "outputs": [ + [ + [ + 3.296875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.47.feed_forward.router": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "outputs": [ + [ + [ + 556.0 + ] + ], + [ + [ + 99.5 + ] + ], + [ + [ + 100.0 + ] + ], + [ + [ + 201.0 + ] + ], + [ + [ + 202.0 + ] + ], + [ + [ + 79.5 + ] + ], + [ + [ + 93.5 + ] + ], + [ + [ + 300.0 + ] + ], + [ + [ + 226.0 + ] + ], + [ + [ + 34.75 + ] + ], + [ + [ + 175.0 + ] + ], + [ + [ + 290.0 + ] + ], + [ + [ + 114.5 + ] + ], + [ + [ + 108.0 + ] + ], + [ + [ + 278.0 + ] + ], + [ + [ + 438.0 + ] + ], + [ + [ + 81.0 + ] + ] + ] + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1083984375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.80078125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.83984375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.83203125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6015625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.79296875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5390625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8359375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.75390625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.55078125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.546875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7265625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.90625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.74609375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40234375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 928.0 + ] + ] + ], + "outputs": [ + [ + [ + 1184.0 + ] + ], + [ + [ + 2.222952092560223e+34 + ] + ] + ], + "params": { + "weight": [ + [ + 0.56640625 + ] + ] + } + }, + "language_model.lm_head": { + "inputs": [ + [ + [ + 54.25 + ] + ] + ], + "params": { + "weight": [ + [ + 1.0859375 + ] + ] + } + } + } +} \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_2_8.npz b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_2_8.npz new file mode 100644 index 000000000000..83c8bb089ce9 Binary files /dev/null and b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_2_8.npz differ diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_2_8_mod_list.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_2_8_mod_list.json new file mode 100644 index 000000000000..449fce78b77d --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_2_8_mod_list.json @@ -0,0 +1,1559 @@ +[ + "vision_model.patch_embedding.linear", + "vision_model.model.layers.0.self_attn.qkv_proj", + "vision_model.model.layers.0.self_attn.o_proj", + "vision_model.model.layers.0.mlp.fc1", + "vision_model.model.layers.0.mlp.fc2", + "vision_model.model.layers.1.self_attn.qkv_proj", + "vision_model.model.layers.1.self_attn.o_proj", + "vision_model.model.layers.1.mlp.fc1", + "vision_model.model.layers.1.mlp.fc2", + "vision_model.model.layers.2.self_attn.qkv_proj", + "vision_model.model.layers.2.self_attn.o_proj", + "vision_model.model.layers.2.mlp.fc1", + "vision_model.model.layers.2.mlp.fc2", + "vision_model.model.layers.3.self_attn.qkv_proj", + "vision_model.model.layers.3.self_attn.o_proj", + "vision_model.model.layers.3.mlp.fc1", + "vision_model.model.layers.3.mlp.fc2", + "vision_model.model.layers.4.self_attn.qkv_proj", + "vision_model.model.layers.4.self_attn.o_proj", + "vision_model.model.layers.4.mlp.fc1", + "vision_model.model.layers.4.mlp.fc2", + "vision_model.model.layers.5.self_attn.qkv_proj", + "vision_model.model.layers.5.self_attn.o_proj", + "vision_model.model.layers.5.mlp.fc1", + "vision_model.model.layers.5.mlp.fc2", + "vision_model.model.layers.6.self_attn.qkv_proj", + "vision_model.model.layers.6.self_attn.o_proj", + "vision_model.model.layers.6.mlp.fc1", + "vision_model.model.layers.6.mlp.fc2", + "vision_model.model.layers.7.self_attn.qkv_proj", + "vision_model.model.layers.7.self_attn.o_proj", + "vision_model.model.layers.7.mlp.fc1", + "vision_model.model.layers.7.mlp.fc2", + "vision_model.model.layers.8.self_attn.qkv_proj", + "vision_model.model.layers.8.self_attn.o_proj", + "vision_model.model.layers.8.mlp.fc1", + "vision_model.model.layers.8.mlp.fc2", + "vision_model.model.layers.9.self_attn.qkv_proj", + "vision_model.model.layers.9.self_attn.o_proj", + "vision_model.model.layers.9.mlp.fc1", + "vision_model.model.layers.9.mlp.fc2", + "vision_model.model.layers.10.self_attn.qkv_proj", + "vision_model.model.layers.10.self_attn.o_proj", + "vision_model.model.layers.10.mlp.fc1", + "vision_model.model.layers.10.mlp.fc2", + "vision_model.model.layers.11.self_attn.qkv_proj", + "vision_model.model.layers.11.self_attn.o_proj", + "vision_model.model.layers.11.mlp.fc1", + "vision_model.model.layers.11.mlp.fc2", + "vision_model.model.layers.12.self_attn.qkv_proj", + "vision_model.model.layers.12.self_attn.o_proj", + "vision_model.model.layers.12.mlp.fc1", + "vision_model.model.layers.12.mlp.fc2", + "vision_model.model.layers.13.self_attn.qkv_proj", + "vision_model.model.layers.13.self_attn.o_proj", + "vision_model.model.layers.13.mlp.fc1", + "vision_model.model.layers.13.mlp.fc2", + "vision_model.model.layers.14.self_attn.qkv_proj", + "vision_model.model.layers.14.self_attn.o_proj", + "vision_model.model.layers.14.mlp.fc1", + "vision_model.model.layers.14.mlp.fc2", + "vision_model.model.layers.15.self_attn.qkv_proj", + "vision_model.model.layers.15.self_attn.o_proj", + "vision_model.model.layers.15.mlp.fc1", + "vision_model.model.layers.15.mlp.fc2", + "vision_model.model.layers.16.self_attn.qkv_proj", + "vision_model.model.layers.16.self_attn.o_proj", + "vision_model.model.layers.16.mlp.fc1", + "vision_model.model.layers.16.mlp.fc2", + "vision_model.model.layers.17.self_attn.qkv_proj", + "vision_model.model.layers.17.self_attn.o_proj", + "vision_model.model.layers.17.mlp.fc1", + "vision_model.model.layers.17.mlp.fc2", + "vision_model.model.layers.18.self_attn.qkv_proj", + "vision_model.model.layers.18.self_attn.o_proj", + "vision_model.model.layers.18.mlp.fc1", + "vision_model.model.layers.18.mlp.fc2", + "vision_model.model.layers.19.self_attn.qkv_proj", + "vision_model.model.layers.19.self_attn.o_proj", + "vision_model.model.layers.19.mlp.fc1", + "vision_model.model.layers.19.mlp.fc2", + "vision_model.model.layers.20.self_attn.qkv_proj", + "vision_model.model.layers.20.self_attn.o_proj", + "vision_model.model.layers.20.mlp.fc1", + "vision_model.model.layers.20.mlp.fc2", + "vision_model.model.layers.21.self_attn.qkv_proj", + "vision_model.model.layers.21.self_attn.o_proj", + "vision_model.model.layers.21.mlp.fc1", + "vision_model.model.layers.21.mlp.fc2", + "vision_model.model.layers.22.self_attn.qkv_proj", + "vision_model.model.layers.22.self_attn.o_proj", + "vision_model.model.layers.22.mlp.fc1", + "vision_model.model.layers.22.mlp.fc2", + "vision_model.model.layers.23.self_attn.qkv_proj", + "vision_model.model.layers.23.self_attn.o_proj", + "vision_model.model.layers.23.mlp.fc1", + "vision_model.model.layers.23.mlp.fc2", + "vision_model.model.layers.24.self_attn.qkv_proj", + "vision_model.model.layers.24.self_attn.o_proj", + "vision_model.model.layers.24.mlp.fc1", + "vision_model.model.layers.24.mlp.fc2", + "vision_model.model.layers.25.self_attn.qkv_proj", + "vision_model.model.layers.25.self_attn.o_proj", + "vision_model.model.layers.25.mlp.fc1", + "vision_model.model.layers.25.mlp.fc2", + "vision_model.model.layers.26.self_attn.qkv_proj", + "vision_model.model.layers.26.self_attn.o_proj", + "vision_model.model.layers.26.mlp.fc1", + "vision_model.model.layers.26.mlp.fc2", + "vision_model.model.layers.27.self_attn.qkv_proj", + "vision_model.model.layers.27.self_attn.o_proj", + "vision_model.model.layers.27.mlp.fc1", + "vision_model.model.layers.27.mlp.fc2", + "vision_model.model.layers.28.self_attn.qkv_proj", + "vision_model.model.layers.28.self_attn.o_proj", + "vision_model.model.layers.28.mlp.fc1", + "vision_model.model.layers.28.mlp.fc2", + "vision_model.model.layers.29.self_attn.qkv_proj", + "vision_model.model.layers.29.self_attn.o_proj", + "vision_model.model.layers.29.mlp.fc1", + "vision_model.model.layers.29.mlp.fc2", + "vision_model.model.layers.30.self_attn.qkv_proj", + "vision_model.model.layers.30.self_attn.o_proj", + "vision_model.model.layers.30.mlp.fc1", + "vision_model.model.layers.30.mlp.fc2", + "vision_model.model.layers.31.self_attn.qkv_proj", + "vision_model.model.layers.31.self_attn.o_proj", + "vision_model.model.layers.31.mlp.fc1", + "vision_model.model.layers.31.mlp.fc2", + "vision_model.model.layers.32.self_attn.qkv_proj", + "vision_model.model.layers.32.self_attn.o_proj", + "vision_model.model.layers.32.mlp.fc1", + "vision_model.model.layers.32.mlp.fc2", + "vision_model.model.layers.33.self_attn.qkv_proj", + "vision_model.model.layers.33.self_attn.o_proj", + "vision_model.model.layers.33.mlp.fc1", + "vision_model.model.layers.33.mlp.fc2", + "vision_model.vision_adapter.mlp.fc1", + "vision_model.vision_adapter.mlp.fc2", + "multi_modal_projector.linear_1", + "language_model.model.layers.0.self_attn.qkv_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.0.self_attn.attn.impl.softmax", + "language_model.model.layers.0.self_attn.attn.impl.matmul_av", + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.0.self_attn.attn.impl.k_cache", + "language_model.model.layers.0.self_attn.attn.impl.v_cache", + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.0.feed_forward.gate_up_proj", + "language_model.model.layers.0.feed_forward.down_proj", + "language_model.model.layers.1.self_attn.qkv_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.1.self_attn.attn.impl.softmax", + "language_model.model.layers.1.self_attn.attn.impl.matmul_av", + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.1.self_attn.attn.impl.k_cache", + "language_model.model.layers.1.self_attn.attn.impl.v_cache", + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.1.feed_forward.router", + "language_model.model.layers.1.feed_forward.experts", + "language_model.model.layers.1.feed_forward.experts.moe_op", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.1.feed_forward.shared_expert.down_proj", + "language_model.model.layers.2.self_attn.qkv_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.2.self_attn.attn.impl.softmax", + "language_model.model.layers.2.self_attn.attn.impl.matmul_av", + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.2.self_attn.attn.impl.k_cache", + "language_model.model.layers.2.self_attn.attn.impl.v_cache", + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.2.feed_forward.gate_up_proj", + "language_model.model.layers.2.feed_forward.down_proj", + "language_model.model.layers.3.self_attn.qkv_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.3.self_attn.attn.impl.softmax", + "language_model.model.layers.3.self_attn.attn.impl.matmul_av", + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.3.self_attn.attn.impl.k_cache", + "language_model.model.layers.3.self_attn.attn.impl.v_cache", + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.3.feed_forward.router", + "language_model.model.layers.3.feed_forward.experts", + "language_model.model.layers.3.feed_forward.experts.moe_op", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.3.feed_forward.shared_expert.down_proj", + "language_model.model.layers.4.self_attn.qkv_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.4.self_attn.attn.impl.softmax", + "language_model.model.layers.4.self_attn.attn.impl.matmul_av", + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.4.self_attn.attn.impl.k_cache", + "language_model.model.layers.4.self_attn.attn.impl.v_cache", + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.4.feed_forward.gate_up_proj", + "language_model.model.layers.4.feed_forward.down_proj", + "language_model.model.layers.5.self_attn.qkv_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.5.self_attn.attn.impl.softmax", + "language_model.model.layers.5.self_attn.attn.impl.matmul_av", + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.5.self_attn.attn.impl.k_cache", + "language_model.model.layers.5.self_attn.attn.impl.v_cache", + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.5.feed_forward.router", + "language_model.model.layers.5.feed_forward.experts", + "language_model.model.layers.5.feed_forward.experts.moe_op", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.5.feed_forward.shared_expert.down_proj", + "language_model.model.layers.6.self_attn.qkv_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.6.self_attn.attn.impl.softmax", + "language_model.model.layers.6.self_attn.attn.impl.matmul_av", + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.6.self_attn.attn.impl.k_cache", + "language_model.model.layers.6.self_attn.attn.impl.v_cache", + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.6.feed_forward.gate_up_proj", + "language_model.model.layers.6.feed_forward.down_proj", + "language_model.model.layers.7.self_attn.qkv_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.7.self_attn.attn.impl.softmax", + "language_model.model.layers.7.self_attn.attn.impl.matmul_av", + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.7.self_attn.attn.impl.k_cache", + "language_model.model.layers.7.self_attn.attn.impl.v_cache", + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.7.feed_forward.router", + "language_model.model.layers.7.feed_forward.experts", + "language_model.model.layers.7.feed_forward.experts.moe_op", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.7.feed_forward.shared_expert.down_proj", + "language_model.model.layers.8.self_attn.qkv_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.8.self_attn.attn.impl.softmax", + "language_model.model.layers.8.self_attn.attn.impl.matmul_av", + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.8.self_attn.attn.impl.k_cache", + "language_model.model.layers.8.self_attn.attn.impl.v_cache", + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.8.feed_forward.gate_up_proj", + "language_model.model.layers.8.feed_forward.down_proj", + "language_model.model.layers.9.self_attn.qkv_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.9.self_attn.attn.impl.softmax", + "language_model.model.layers.9.self_attn.attn.impl.matmul_av", + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.9.self_attn.attn.impl.k_cache", + "language_model.model.layers.9.self_attn.attn.impl.v_cache", + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.9.feed_forward.router", + "language_model.model.layers.9.feed_forward.experts", + "language_model.model.layers.9.feed_forward.experts.moe_op", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.9.feed_forward.shared_expert.down_proj", + "language_model.model.layers.10.self_attn.qkv_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.10.self_attn.attn.impl.softmax", + "language_model.model.layers.10.self_attn.attn.impl.matmul_av", + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.10.self_attn.attn.impl.k_cache", + "language_model.model.layers.10.self_attn.attn.impl.v_cache", + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.10.feed_forward.gate_up_proj", + "language_model.model.layers.10.feed_forward.down_proj", + "language_model.model.layers.11.self_attn.qkv_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.11.self_attn.attn.impl.softmax", + "language_model.model.layers.11.self_attn.attn.impl.matmul_av", + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.11.self_attn.attn.impl.k_cache", + "language_model.model.layers.11.self_attn.attn.impl.v_cache", + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.11.feed_forward.router", + "language_model.model.layers.11.feed_forward.experts", + "language_model.model.layers.11.feed_forward.experts.moe_op", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.11.feed_forward.shared_expert.down_proj", + "language_model.model.layers.12.self_attn.qkv_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.12.self_attn.attn.impl.softmax", + "language_model.model.layers.12.self_attn.attn.impl.matmul_av", + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.12.self_attn.attn.impl.k_cache", + "language_model.model.layers.12.self_attn.attn.impl.v_cache", + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.12.feed_forward.gate_up_proj", + "language_model.model.layers.12.feed_forward.down_proj", + "language_model.model.layers.13.self_attn.qkv_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.13.self_attn.attn.impl.softmax", + "language_model.model.layers.13.self_attn.attn.impl.matmul_av", + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.13.self_attn.attn.impl.k_cache", + "language_model.model.layers.13.self_attn.attn.impl.v_cache", + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.13.feed_forward.router", + "language_model.model.layers.13.feed_forward.experts", + "language_model.model.layers.13.feed_forward.experts.moe_op", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.13.feed_forward.shared_expert.down_proj", + "language_model.model.layers.14.self_attn.qkv_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.14.self_attn.attn.impl.softmax", + "language_model.model.layers.14.self_attn.attn.impl.matmul_av", + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.14.self_attn.attn.impl.k_cache", + "language_model.model.layers.14.self_attn.attn.impl.v_cache", + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.14.feed_forward.gate_up_proj", + "language_model.model.layers.14.feed_forward.down_proj", + "language_model.model.layers.15.self_attn.qkv_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.15.self_attn.attn.impl.softmax", + "language_model.model.layers.15.self_attn.attn.impl.matmul_av", + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.15.self_attn.attn.impl.k_cache", + "language_model.model.layers.15.self_attn.attn.impl.v_cache", + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.15.feed_forward.router", + "language_model.model.layers.15.feed_forward.experts", + "language_model.model.layers.15.feed_forward.experts.moe_op", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.15.feed_forward.shared_expert.down_proj", + "language_model.model.layers.16.self_attn.qkv_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.16.self_attn.attn.impl.softmax", + "language_model.model.layers.16.self_attn.attn.impl.matmul_av", + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.16.self_attn.attn.impl.k_cache", + "language_model.model.layers.16.self_attn.attn.impl.v_cache", + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.16.feed_forward.gate_up_proj", + "language_model.model.layers.16.feed_forward.down_proj", + "language_model.model.layers.17.self_attn.qkv_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.17.self_attn.attn.impl.softmax", + "language_model.model.layers.17.self_attn.attn.impl.matmul_av", + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.17.self_attn.attn.impl.k_cache", + "language_model.model.layers.17.self_attn.attn.impl.v_cache", + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.17.feed_forward.router", + "language_model.model.layers.17.feed_forward.experts", + "language_model.model.layers.17.feed_forward.experts.moe_op", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.17.feed_forward.shared_expert.down_proj", + "language_model.model.layers.18.self_attn.qkv_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.18.self_attn.attn.impl.softmax", + "language_model.model.layers.18.self_attn.attn.impl.matmul_av", + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.18.self_attn.attn.impl.k_cache", + "language_model.model.layers.18.self_attn.attn.impl.v_cache", + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.18.feed_forward.gate_up_proj", + "language_model.model.layers.18.feed_forward.down_proj", + "language_model.model.layers.19.self_attn.qkv_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.19.self_attn.attn.impl.softmax", + "language_model.model.layers.19.self_attn.attn.impl.matmul_av", + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.19.self_attn.attn.impl.k_cache", + "language_model.model.layers.19.self_attn.attn.impl.v_cache", + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.19.feed_forward.router", + "language_model.model.layers.19.feed_forward.experts", + "language_model.model.layers.19.feed_forward.experts.moe_op", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.19.feed_forward.shared_expert.down_proj", + "language_model.model.layers.20.self_attn.qkv_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.20.self_attn.attn.impl.softmax", + "language_model.model.layers.20.self_attn.attn.impl.matmul_av", + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.20.self_attn.attn.impl.k_cache", + "language_model.model.layers.20.self_attn.attn.impl.v_cache", + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.20.feed_forward.gate_up_proj", + "language_model.model.layers.20.feed_forward.down_proj", + "language_model.model.layers.21.self_attn.qkv_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.21.self_attn.attn.impl.softmax", + "language_model.model.layers.21.self_attn.attn.impl.matmul_av", + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.21.self_attn.attn.impl.k_cache", + "language_model.model.layers.21.self_attn.attn.impl.v_cache", + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.21.feed_forward.router", + "language_model.model.layers.21.feed_forward.experts", + "language_model.model.layers.21.feed_forward.experts.moe_op", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.21.feed_forward.shared_expert.down_proj", + "language_model.model.layers.22.self_attn.qkv_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.22.self_attn.attn.impl.softmax", + "language_model.model.layers.22.self_attn.attn.impl.matmul_av", + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.22.self_attn.attn.impl.k_cache", + "language_model.model.layers.22.self_attn.attn.impl.v_cache", + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.22.feed_forward.gate_up_proj", + "language_model.model.layers.22.feed_forward.down_proj", + "language_model.model.layers.23.self_attn.qkv_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.23.self_attn.attn.impl.softmax", + "language_model.model.layers.23.self_attn.attn.impl.matmul_av", + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.23.self_attn.attn.impl.k_cache", + "language_model.model.layers.23.self_attn.attn.impl.v_cache", + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.23.feed_forward.router", + "language_model.model.layers.23.feed_forward.experts", + "language_model.model.layers.23.feed_forward.experts.moe_op", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.23.feed_forward.shared_expert.down_proj", + "language_model.model.layers.24.self_attn.qkv_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.24.self_attn.attn.impl.softmax", + "language_model.model.layers.24.self_attn.attn.impl.matmul_av", + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.24.self_attn.attn.impl.k_cache", + "language_model.model.layers.24.self_attn.attn.impl.v_cache", + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.24.feed_forward.gate_up_proj", + "language_model.model.layers.24.feed_forward.down_proj", + "language_model.model.layers.25.self_attn.qkv_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.25.self_attn.attn.impl.softmax", + "language_model.model.layers.25.self_attn.attn.impl.matmul_av", + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.25.self_attn.attn.impl.k_cache", + "language_model.model.layers.25.self_attn.attn.impl.v_cache", + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.25.feed_forward.router", + "language_model.model.layers.25.feed_forward.experts", + "language_model.model.layers.25.feed_forward.experts.moe_op", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.25.feed_forward.shared_expert.down_proj", + "language_model.model.layers.26.self_attn.qkv_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.26.self_attn.attn.impl.softmax", + "language_model.model.layers.26.self_attn.attn.impl.matmul_av", + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.26.self_attn.attn.impl.k_cache", + "language_model.model.layers.26.self_attn.attn.impl.v_cache", + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.26.feed_forward.gate_up_proj", + "language_model.model.layers.26.feed_forward.down_proj", + "language_model.model.layers.27.self_attn.qkv_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.27.self_attn.attn.impl.softmax", + "language_model.model.layers.27.self_attn.attn.impl.matmul_av", + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.27.self_attn.attn.impl.k_cache", + "language_model.model.layers.27.self_attn.attn.impl.v_cache", + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.27.feed_forward.router", + "language_model.model.layers.27.feed_forward.experts", + "language_model.model.layers.27.feed_forward.experts.moe_op", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.27.feed_forward.shared_expert.down_proj", + "language_model.model.layers.28.self_attn.qkv_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.28.self_attn.attn.impl.softmax", + "language_model.model.layers.28.self_attn.attn.impl.matmul_av", + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.28.self_attn.attn.impl.k_cache", + "language_model.model.layers.28.self_attn.attn.impl.v_cache", + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.28.feed_forward.gate_up_proj", + "language_model.model.layers.28.feed_forward.down_proj", + "language_model.model.layers.29.self_attn.qkv_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.29.self_attn.attn.impl.softmax", + "language_model.model.layers.29.self_attn.attn.impl.matmul_av", + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.29.self_attn.attn.impl.k_cache", + "language_model.model.layers.29.self_attn.attn.impl.v_cache", + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.29.feed_forward.router", + "language_model.model.layers.29.feed_forward.experts", + "language_model.model.layers.29.feed_forward.experts.moe_op", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.29.feed_forward.shared_expert.down_proj", + "language_model.model.layers.30.self_attn.qkv_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.30.self_attn.attn.impl.softmax", + "language_model.model.layers.30.self_attn.attn.impl.matmul_av", + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.30.self_attn.attn.impl.k_cache", + "language_model.model.layers.30.self_attn.attn.impl.v_cache", + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.30.feed_forward.gate_up_proj", + "language_model.model.layers.30.feed_forward.down_proj", + "language_model.model.layers.31.self_attn.qkv_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.31.self_attn.attn.impl.softmax", + "language_model.model.layers.31.self_attn.attn.impl.matmul_av", + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.31.self_attn.attn.impl.k_cache", + "language_model.model.layers.31.self_attn.attn.impl.v_cache", + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.31.feed_forward.router", + "language_model.model.layers.31.feed_forward.experts", + "language_model.model.layers.31.feed_forward.experts.moe_op", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.31.feed_forward.shared_expert.down_proj", + "language_model.model.layers.32.self_attn.qkv_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.32.self_attn.attn.impl.softmax", + "language_model.model.layers.32.self_attn.attn.impl.matmul_av", + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.32.self_attn.attn.impl.k_cache", + "language_model.model.layers.32.self_attn.attn.impl.v_cache", + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.32.feed_forward.gate_up_proj", + "language_model.model.layers.32.feed_forward.down_proj", + "language_model.model.layers.33.self_attn.qkv_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.33.self_attn.attn.impl.softmax", + "language_model.model.layers.33.self_attn.attn.impl.matmul_av", + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.33.self_attn.attn.impl.k_cache", + "language_model.model.layers.33.self_attn.attn.impl.v_cache", + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.33.feed_forward.router", + "language_model.model.layers.33.feed_forward.experts", + "language_model.model.layers.33.feed_forward.experts.moe_op", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.33.feed_forward.shared_expert.down_proj", + "language_model.model.layers.34.self_attn.qkv_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.34.self_attn.attn.impl.softmax", + "language_model.model.layers.34.self_attn.attn.impl.matmul_av", + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.34.self_attn.attn.impl.k_cache", + "language_model.model.layers.34.self_attn.attn.impl.v_cache", + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.34.feed_forward.gate_up_proj", + "language_model.model.layers.34.feed_forward.down_proj", + "language_model.model.layers.35.self_attn.qkv_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.35.self_attn.attn.impl.softmax", + "language_model.model.layers.35.self_attn.attn.impl.matmul_av", + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.35.self_attn.attn.impl.k_cache", + "language_model.model.layers.35.self_attn.attn.impl.v_cache", + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.35.feed_forward.router", + "language_model.model.layers.35.feed_forward.experts", + "language_model.model.layers.35.feed_forward.experts.moe_op", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.35.feed_forward.shared_expert.down_proj", + "language_model.model.layers.36.self_attn.qkv_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.36.self_attn.attn.impl.softmax", + "language_model.model.layers.36.self_attn.attn.impl.matmul_av", + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.36.self_attn.attn.impl.k_cache", + "language_model.model.layers.36.self_attn.attn.impl.v_cache", + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.36.feed_forward.gate_up_proj", + "language_model.model.layers.36.feed_forward.down_proj", + "language_model.model.layers.37.self_attn.qkv_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.37.self_attn.attn.impl.softmax", + "language_model.model.layers.37.self_attn.attn.impl.matmul_av", + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.37.self_attn.attn.impl.k_cache", + "language_model.model.layers.37.self_attn.attn.impl.v_cache", + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.37.feed_forward.router", + "language_model.model.layers.37.feed_forward.experts", + "language_model.model.layers.37.feed_forward.experts.moe_op", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.37.feed_forward.shared_expert.down_proj", + "language_model.model.layers.38.self_attn.qkv_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.38.self_attn.attn.impl.softmax", + "language_model.model.layers.38.self_attn.attn.impl.matmul_av", + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.38.self_attn.attn.impl.k_cache", + "language_model.model.layers.38.self_attn.attn.impl.v_cache", + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.38.feed_forward.gate_up_proj", + "language_model.model.layers.38.feed_forward.down_proj", + "language_model.model.layers.39.self_attn.qkv_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.39.self_attn.attn.impl.softmax", + "language_model.model.layers.39.self_attn.attn.impl.matmul_av", + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.39.self_attn.attn.impl.k_cache", + "language_model.model.layers.39.self_attn.attn.impl.v_cache", + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.39.feed_forward.router", + "language_model.model.layers.39.feed_forward.experts", + "language_model.model.layers.39.feed_forward.experts.moe_op", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.39.feed_forward.shared_expert.down_proj", + "language_model.model.layers.40.self_attn.qkv_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.40.self_attn.attn.impl.softmax", + "language_model.model.layers.40.self_attn.attn.impl.matmul_av", + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.40.self_attn.attn.impl.k_cache", + "language_model.model.layers.40.self_attn.attn.impl.v_cache", + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.40.feed_forward.gate_up_proj", + "language_model.model.layers.40.feed_forward.down_proj", + "language_model.model.layers.41.self_attn.qkv_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.41.self_attn.attn.impl.softmax", + "language_model.model.layers.41.self_attn.attn.impl.matmul_av", + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.41.self_attn.attn.impl.k_cache", + "language_model.model.layers.41.self_attn.attn.impl.v_cache", + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.41.feed_forward.router", + "language_model.model.layers.41.feed_forward.experts", + "language_model.model.layers.41.feed_forward.experts.moe_op", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.41.feed_forward.shared_expert.down_proj", + "language_model.model.layers.42.self_attn.qkv_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.42.self_attn.attn.impl.softmax", + "language_model.model.layers.42.self_attn.attn.impl.matmul_av", + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.42.self_attn.attn.impl.k_cache", + "language_model.model.layers.42.self_attn.attn.impl.v_cache", + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.42.feed_forward.gate_up_proj", + "language_model.model.layers.42.feed_forward.down_proj", + "language_model.model.layers.43.self_attn.qkv_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.43.self_attn.attn.impl.softmax", + "language_model.model.layers.43.self_attn.attn.impl.matmul_av", + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.43.self_attn.attn.impl.k_cache", + "language_model.model.layers.43.self_attn.attn.impl.v_cache", + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.43.feed_forward.router", + "language_model.model.layers.43.feed_forward.experts", + "language_model.model.layers.43.feed_forward.experts.moe_op", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.43.feed_forward.shared_expert.down_proj", + "language_model.model.layers.44.self_attn.qkv_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.44.self_attn.attn.impl.softmax", + "language_model.model.layers.44.self_attn.attn.impl.matmul_av", + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.44.self_attn.attn.impl.k_cache", + "language_model.model.layers.44.self_attn.attn.impl.v_cache", + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.44.feed_forward.gate_up_proj", + "language_model.model.layers.44.feed_forward.down_proj", + "language_model.model.layers.45.self_attn.qkv_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.45.self_attn.attn.impl.softmax", + "language_model.model.layers.45.self_attn.attn.impl.matmul_av", + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.45.self_attn.attn.impl.k_cache", + "language_model.model.layers.45.self_attn.attn.impl.v_cache", + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.45.feed_forward.router", + "language_model.model.layers.45.feed_forward.experts", + "language_model.model.layers.45.feed_forward.experts.moe_op", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.45.feed_forward.shared_expert.down_proj", + "language_model.model.layers.46.self_attn.qkv_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.46.self_attn.attn.impl.softmax", + "language_model.model.layers.46.self_attn.attn.impl.matmul_av", + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.46.self_attn.attn.impl.k_cache", + "language_model.model.layers.46.self_attn.attn.impl.v_cache", + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.46.feed_forward.gate_up_proj", + "language_model.model.layers.46.feed_forward.down_proj", + "language_model.model.layers.47.self_attn.qkv_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.47.self_attn.attn.impl.softmax", + "language_model.model.layers.47.self_attn.attn.impl.matmul_av", + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.47.self_attn.attn.impl.k_cache", + "language_model.model.layers.47.self_attn.attn.impl.v_cache", + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.47.feed_forward.router", + "language_model.model.layers.47.feed_forward.experts", + "language_model.model.layers.47.feed_forward.experts.moe_op", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.47.feed_forward.shared_expert.down_proj", + "language_model.lm_head" +] \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_3_8.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_3_8.json new file mode 100644 index 000000000000..12c7e44c5e62 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_3_8.json @@ -0,0 +1,24263 @@ +{ + "GlobalRank": null, + "LocalRank": 3, + "Mode": "DynamicRange", + "Nodes": { + "language_model.model.layers.0.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 11.1875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.0.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.765625 + ] + ] + ], + "outputs": [ + [ + [ + 0.83203125 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.345703125 + ] + ] + } + }, + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.41796875 + ] + ], + [ + [ + 5.25 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.91015625 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.41796875 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.474609375 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 5.25 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.91015625 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 5.03125 + ] + ], + [ + [ + 5.25 + ] + ], + [ + [ + 0.91015625 + ] + ] + ], + "outputs": [ + [ + [ + 0.765625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.0.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 2.546875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.0.feed_forward.down_proj": { + "inputs": [ + [ + [ + 22.375 + ] + ] + ], + "outputs": [ + [ + [ + 26.75 + ] + ], + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.765625 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 10.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.4609375 + ] + ] + ], + "outputs": [ + [ + [ + 0.67578125 + ] + ], + [ + [ + 1.3984375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2353515625 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.82421875 + ] + ], + [ + [ + 9.5625 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.7265625 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.82421875 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.40625 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 9.5625 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.7265625 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.375 + ] + ], + [ + [ + 9.5625 + ] + ], + [ + [ + 1.7265625 + ] + ] + ], + "outputs": [ + [ + [ + 1.4609375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.1.feed_forward.router": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "outputs": [ + [ + [ + 1.5390625 + ] + ], + [ + [ + 22.25 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 16.75 + ] + ], + [ + [ + 13.125 + ] + ], + [ + [ + 16.625 + ] + ], + [ + [ + 17.25 + ] + ], + [ + [ + 20.875 + ] + ], + [ + [ + 17.25 + ] + ], + [ + [ + 15.3125 + ] + ], + [ + [ + 7.9375 + ] + ], + [ + [ + 13.625 + ] + ], + [ + [ + 14.125 + ] + ], + [ + [ + 14.125 + ] + ], + [ + [ + 17.875 + ] + ], + [ + [ + 18.625 + ] + ], + [ + [ + 0.0 + ] + ] + ] + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11181640625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.35546875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.240234375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08984375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23828125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 5.4375 + ] + ] + ], + "outputs": [ + [ + [ + 0.33203125 + ] + ], + [ + [ + 8.639981972601953e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 9.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.8828125 + ] + ] + ], + "outputs": [ + [ + [ + 0.7734375 + ] + ], + [ + [ + 1.4921875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.609375 + ] + ], + [ + [ + 8.875 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.140625 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.609375 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.76953125 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 8.875 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.140625 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.5625 + ] + ], + [ + [ + 8.875 + ] + ], + [ + [ + 1.140625 + ] + ] + ], + "outputs": [ + [ + [ + 0.8828125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.2.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 6.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.2.feed_forward.down_proj": { + "inputs": [ + [ + [ + 5.03125 + ] + ] + ], + "outputs": [ + [ + [ + 4.75 + ] + ], + [ + [ + 25.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33203125 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 7.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.41796875 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.2890625 + ] + ] + ], + "outputs": [ + [ + [ + 0.77734375 + ] + ], + [ + [ + 2.40625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.173828125 + ] + ], + [ + [ + 4.0625 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.33203125 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.173828125 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.169921875 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 4.0625 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.33203125 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 2.03125 + ] + ], + [ + [ + 4.0625 + ] + ], + [ + [ + 0.33203125 + ] + ] + ], + "outputs": [ + [ + [ + 0.2890625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.3.feed_forward.router": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.52734375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "outputs": [ + [ + [ + 2.09375 + ] + ], + [ + [ + 23.875 + ] + ], + [ + [ + 16.5 + ] + ], + [ + [ + 17.25 + ] + ], + [ + [ + 27.625 + ] + ], + [ + [ + 31.75 + ] + ], + [ + [ + 8.3125 + ] + ], + [ + [ + 14.875 + ] + ], + [ + [ + 21.625 + ] + ], + [ + [ + 16.0 + ] + ], + [ + [ + 19.875 + ] + ], + [ + [ + 21.125 + ] + ], + [ + [ + 24.375 + ] + ], + [ + [ + 4.78125 + ] + ], + [ + [ + 20.125 + ] + ], + [ + [ + 20.375 + ] + ], + [ + [ + 38.0 + ] + ] + ] + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10888671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09326171875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10986328125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08544921875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08447265625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 5.78125 + ] + ] + ], + "outputs": [ + [ + [ + 0.921875 + ] + ], + [ + [ + 5.0469125464958525e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.349609375 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 14.3125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.28125 + ] + ] + ], + "outputs": [ + [ + [ + 1.46875 + ] + ], + [ + [ + 2.328125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.78515625 + ] + ], + [ + [ + 12.875 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.9375 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.78515625 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.25 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 12.875 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.9375 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.4375 + ] + ], + [ + [ + 12.4375 + ] + ], + [ + [ + 2.9375 + ] + ] + ], + "outputs": [ + [ + [ + 2.28125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.4.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 6.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.322265625 + ] + ] + } + }, + "language_model.model.layers.4.feed_forward.down_proj": { + "inputs": [ + [ + [ + 6.59375 + ] + ] + ], + "outputs": [ + [ + [ + 2.796875 + ] + ], + [ + [ + 5.1875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.482421875 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 18.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.515625 + ] + ] + ], + "outputs": [ + [ + [ + 1.4921875 + ] + ], + [ + [ + 2.40625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.310546875 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.77734375 + ] + ], + [ + [ + 18.875 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.1875 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.77734375 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.953125 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.875 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.1875 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.3125 + ] + ], + [ + [ + 18.75 + ] + ], + [ + [ + 5.1875 + ] + ] + ], + "outputs": [ + [ + [ + 2.515625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.5.feed_forward.router": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 2.8125 + ] + ], + [ + [ + 7.09375 + ] + ], + [ + [ + 23.375 + ] + ], + [ + [ + 24.75 + ] + ], + [ + [ + 20.625 + ] + ], + [ + [ + 24.75 + ] + ], + [ + [ + 12.125 + ] + ], + [ + [ + 13.5 + ] + ], + [ + [ + 14.6875 + ] + ], + [ + [ + 16.75 + ] + ], + [ + [ + 50.5 + ] + ], + [ + [ + 31.375 + ] + ], + [ + [ + 23.25 + ] + ], + [ + [ + 22.875 + ] + ], + [ + [ + 19.25 + ] + ], + [ + [ + 31.75 + ] + ], + [ + [ + 25.75 + ] + ] + ] + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10888671875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11474609375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1083984375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.322265625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.103515625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1083984375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10693359375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.111328125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0859375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 5.96875 + ] + ] + ], + "outputs": [ + [ + [ + 0.58984375 + ] + ], + [ + [ + 1.0384593717069655e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 14.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.3125 + ] + ] + ], + "outputs": [ + [ + [ + 0.94140625 + ] + ], + [ + [ + 1.96875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.8125 + ] + ], + [ + [ + 17.0 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.09375 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.8125 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.515625 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.0 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.09375 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.4375 + ] + ], + [ + [ + 17.0 + ] + ], + [ + [ + 3.09375 + ] + ] + ], + "outputs": [ + [ + [ + 2.3125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.6.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 7.71875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.6.feed_forward.down_proj": { + "inputs": [ + [ + [ + 7.15625 + ] + ] + ], + "outputs": [ + [ + [ + 2.25 + ] + ], + [ + [ + 11.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.609375 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 7.59375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.0 + ] + ] + ], + "outputs": [ + [ + [ + 2.1875 + ] + ], + [ + [ + 2.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.478515625 + ] + ], + [ + [ + 4.4375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.5234375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.478515625 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.9609375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 4.4375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.5234375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 5.0625 + ] + ], + [ + [ + 4.1875 + ] + ], + [ + [ + 1.5234375 + ] + ] + ], + "outputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.7.feed_forward.router": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "outputs": [ + [ + [ + 3.296875 + ] + ], + [ + [ + 36.5 + ] + ], + [ + [ + 30.25 + ] + ], + [ + [ + 34.25 + ] + ], + [ + [ + 29.625 + ] + ], + [ + [ + 32.75 + ] + ], + [ + [ + 20.625 + ] + ], + [ + [ + 52.25 + ] + ], + [ + [ + 31.375 + ] + ], + [ + [ + 17.625 + ] + ], + [ + [ + 44.0 + ] + ], + [ + [ + 8.1875 + ] + ], + [ + [ + 21.125 + ] + ], + [ + [ + 49.25 + ] + ], + [ + [ + 32.0 + ] + ], + [ + [ + 23.5 + ] + ], + [ + [ + 49.75 + ] + ] + ] + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10888671875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.53515625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "outputs": [ + [ + [ + 1.3828125 + ] + ], + [ + [ + 8.639981972601953e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.6875 + ] + ] + ], + "outputs": [ + [ + [ + 2.265625 + ] + ], + [ + [ + 3.390625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.86328125 + ] + ], + [ + [ + 16.375 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.703125 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.86328125 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.5625 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.375 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.703125 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.875 + ] + ], + [ + [ + 16.375 + ] + ], + [ + [ + 3.6875 + ] + ] + ], + "outputs": [ + [ + [ + 2.6875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.8.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 8.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.8.feed_forward.down_proj": { + "inputs": [ + [ + [ + 9.875 + ] + ] + ], + "outputs": [ + [ + [ + 2.84375 + ] + ], + [ + [ + 9.8125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.59375 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 16.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.4375 + ] + ] + ], + "outputs": [ + [ + [ + 1.609375 + ] + ], + [ + [ + 3.640625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.03125 + ] + ], + [ + [ + 18.125 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.484375 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.03125 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.4375 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.125 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.484375 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.875 + ] + ], + [ + [ + 18.125 + ] + ], + [ + [ + 3.484375 + ] + ] + ], + "outputs": [ + [ + [ + 2.234375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.9.feed_forward.router": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "outputs": [ + [ + [ + 5.3125 + ] + ], + [ + [ + 39.25 + ] + ], + [ + [ + 57.5 + ] + ], + [ + [ + 32.75 + ] + ], + [ + [ + 57.5 + ] + ], + [ + [ + 18.375 + ] + ], + [ + [ + 21.5 + ] + ], + [ + [ + 29.5 + ] + ], + [ + [ + 23.5 + ] + ], + [ + [ + 55.0 + ] + ], + [ + [ + 39.75 + ] + ], + [ + [ + 62.75 + ] + ], + [ + [ + 41.75 + ] + ], + [ + [ + 24.75 + ] + ], + [ + [ + 21.25 + ] + ], + [ + [ + 13.25 + ] + ], + [ + [ + 53.0 + ] + ] + ] + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0869140625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.091796875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0869140625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10693359375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1025390625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 11.6875 + ] + ] + ], + "outputs": [ + [ + [ + 3.09375 + ] + ], + [ + [ + 7.393830726553595e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4375 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 17.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.59375 + ] + ] + ], + "outputs": [ + [ + [ + 1.984375 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.94921875 + ] + ], + [ + [ + 19.75 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.703125 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.94921875 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.078125 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.75 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.703125 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.125 + ] + ], + [ + [ + 19.625 + ] + ], + [ + [ + 3.703125 + ] + ] + ], + "outputs": [ + [ + [ + 2.59375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.10.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 9.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.10.feed_forward.down_proj": { + "inputs": [ + [ + [ + 11.3125 + ] + ] + ], + "outputs": [ + [ + [ + 3.40625 + ] + ], + [ + [ + 12.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.515625 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 15.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.46875 + ] + ] + ], + "outputs": [ + [ + [ + 8.625 + ] + ], + [ + [ + 8.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.96484375 + ] + ], + [ + [ + 11.3125 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.15625 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.96484375 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.90625 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 11.3125 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.15625 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.8125 + ] + ], + [ + [ + 11.3125 + ] + ], + [ + [ + 3.15625 + ] + ] + ], + "outputs": [ + [ + [ + 2.46875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.11.feed_forward.router": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "outputs": [ + [ + [ + 5.21875 + ] + ], + [ + [ + 46.25 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 61.0 + ] + ], + [ + [ + 34.25 + ] + ], + [ + [ + 42.75 + ] + ], + [ + [ + 49.75 + ] + ], + [ + [ + 67.0 + ] + ], + [ + [ + 62.0 + ] + ], + [ + [ + 27.875 + ] + ], + [ + [ + 59.25 + ] + ], + [ + [ + 52.0 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 70.0 + ] + ], + [ + [ + 42.0 + ] + ], + [ + [ + 77.0 + ] + ], + [ + [ + 37.5 + ] + ] + ] + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10888671875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0869140625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08935546875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10888671875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 30.375 + ] + ] + ], + "outputs": [ + [ + [ + 2.078125 + ] + ], + [ + [ + 7.393830726553595e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 20.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.53125 + ] + ] + ], + "outputs": [ + [ + [ + 1.796875 + ] + ], + [ + [ + 3.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.015625 + ] + ], + [ + [ + 19.75 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.09375 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.015625 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.03125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.75 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.09375 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.0 + ] + ], + [ + [ + 17.75 + ] + ], + [ + [ + 4.09375 + ] + ] + ], + "outputs": [ + [ + [ + 3.53125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.12.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 10.6875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.12.feed_forward.down_proj": { + "inputs": [ + [ + [ + 19.0 + ] + ] + ], + "outputs": [ + [ + [ + 9.25 + ] + ], + [ + [ + 20.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6484375 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.9375 + ] + ] + ], + "outputs": [ + [ + [ + 2.015625 + ] + ], + [ + [ + 6.34375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.8046875 + ] + ], + [ + [ + 16.625 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.0 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.8046875 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.34375 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.0 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.3125 + ] + ], + [ + [ + 16.375 + ] + ], + [ + [ + 4.0 + ] + ] + ], + "outputs": [ + [ + [ + 2.9375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.13.feed_forward.router": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "outputs": [ + [ + [ + 5.5625 + ] + ], + [ + [ + 63.25 + ] + ], + [ + [ + 83.0 + ] + ], + [ + [ + 51.5 + ] + ], + [ + [ + 67.0 + ] + ], + [ + [ + 31.75 + ] + ], + [ + [ + 57.5 + ] + ], + [ + [ + 40.75 + ] + ], + [ + [ + 45.5 + ] + ], + [ + [ + 54.5 + ] + ], + [ + [ + 54.25 + ] + ], + [ + [ + 47.25 + ] + ], + [ + [ + 47.75 + ] + ], + [ + [ + 47.5 + ] + ], + [ + [ + 54.75 + ] + ], + [ + [ + 53.0 + ] + ], + [ + [ + 55.25 + ] + ] + ] + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1005859375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1005859375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 13.4375 + ] + ] + ], + "outputs": [ + [ + [ + 3.59375 + ] + ], + [ + [ + 5.0469125464958525e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.55078125 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 20.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.828125 + ] + ] + ], + "outputs": [ + [ + [ + 3.546875 + ] + ], + [ + [ + 5.71875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.90625 + ] + ], + [ + [ + 18.0 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.15625 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.90625 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.515625 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.0 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.15625 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.875 + ] + ], + [ + [ + 17.375 + ] + ], + [ + [ + 3.15625 + ] + ] + ], + "outputs": [ + [ + [ + 2.828125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.14.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 12.0625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.14.feed_forward.down_proj": { + "inputs": [ + [ + [ + 13.6875 + ] + ] + ], + "outputs": [ + [ + [ + 5.375 + ] + ], + [ + [ + 20.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.64453125 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 23.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.890625 + ] + ] + ], + "outputs": [ + [ + [ + 20.0 + ] + ], + [ + [ + 21.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.7890625 + ] + ], + [ + [ + 12.5 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.75 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.7890625 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.375 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 12.5 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.75 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.125 + ] + ], + [ + [ + 12.5 + ] + ], + [ + [ + 4.75 + ] + ] + ], + "outputs": [ + [ + [ + 3.890625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.15.feed_forward.router": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 13.75 + ] + ], + [ + [ + 43.5 + ] + ], + [ + [ + 63.75 + ] + ], + [ + [ + 77.0 + ] + ], + [ + [ + 67.5 + ] + ], + [ + [ + 49.25 + ] + ], + [ + [ + 84.5 + ] + ], + [ + [ + 56.75 + ] + ], + [ + [ + 68.0 + ] + ], + [ + [ + 43.25 + ] + ], + [ + [ + 77.0 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 73.5 + ] + ], + [ + [ + 58.75 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 59.5 + ] + ], + [ + [ + 61.25 + ] + ] + ] + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.51171875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1044921875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10693359375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 20.0 + ] + ] + ], + "outputs": [ + [ + [ + 7.8125 + ] + ], + [ + [ + 8.639981972601953e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.494140625 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 23.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.875 + ] + ] + ], + "outputs": [ + [ + [ + 2.4375 + ] + ], + [ + [ + 6.3125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.9375 + ] + ], + [ + [ + 18.125 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.3125 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.9375 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.953125 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.125 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.3125 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.3125 + ] + ], + [ + [ + 18.125 + ] + ], + [ + [ + 6.3125 + ] + ] + ], + "outputs": [ + [ + [ + 5.875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.16.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 14.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.16.feed_forward.down_proj": { + "inputs": [ + [ + [ + 21.75 + ] + ] + ], + "outputs": [ + [ + [ + 6.65625 + ] + ], + [ + [ + 27.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6640625 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.625 + ] + ] + ], + "outputs": [ + [ + [ + 19.5 + ] + ], + [ + [ + 27.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.87890625 + ] + ], + [ + [ + 10.8125 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.75 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.87890625 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.625 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 10.8125 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.75 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.25 + ] + ], + [ + [ + 10.8125 + ] + ], + [ + [ + 5.75 + ] + ] + ], + "outputs": [ + [ + [ + 4.34375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.17.feed_forward.router": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 89.5 + ] + ], + [ + [ + 58.5 + ] + ], + [ + [ + 35.5 + ] + ], + [ + [ + 83.5 + ] + ], + [ + [ + 93.5 + ] + ], + [ + [ + 28.875 + ] + ], + [ + [ + 81.0 + ] + ], + [ + [ + 64.0 + ] + ], + [ + [ + 61.75 + ] + ], + [ + [ + 74.0 + ] + ], + [ + [ + 43.0 + ] + ], + [ + [ + 63.75 + ] + ], + [ + [ + 58.25 + ] + ], + [ + [ + 84.5 + ] + ], + [ + [ + 74.5 + ] + ], + [ + [ + 53.75 + ] + ], + [ + [ + 40.5 + ] + ] + ] + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6171875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.703125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 57.5 + ] + ] + ], + "outputs": [ + [ + [ + 6.5 + ] + ], + [ + [ + 8.639981972601953e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.25 + ] + ] + ], + "outputs": [ + [ + [ + 6.625 + ] + ], + [ + [ + 12.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.59765625 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.81640625 + ] + ], + [ + [ + 9.625 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.28125 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.81640625 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.0625 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 9.625 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.28125 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.3125 + ] + ], + [ + [ + 9.5 + ] + ], + [ + [ + 6.28125 + ] + ] + ], + "outputs": [ + [ + [ + 5.25 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.18.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 20.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.435546875 + ] + ] + } + }, + "language_model.model.layers.18.feed_forward.down_proj": { + "inputs": [ + [ + [ + 23.25 + ] + ] + ], + "outputs": [ + [ + [ + 8.1875 + ] + ], + [ + [ + 354.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6953125 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 44.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.265625 + ] + ] + ], + "outputs": [ + [ + [ + 31.5 + ] + ], + [ + [ + 37.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.921875 + ] + ], + [ + [ + 10.0 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.53125 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.921875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.234375 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 10.0 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.53125 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.6875 + ] + ], + [ + [ + 10.0 + ] + ], + [ + [ + 6.53125 + ] + ] + ], + "outputs": [ + [ + [ + 3.265625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.19.feed_forward.router": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "outputs": [ + [ + [ + 93.0 + ] + ], + [ + [ + 37.5 + ] + ], + [ + [ + 78.0 + ] + ], + [ + [ + 100.0 + ] + ], + [ + [ + 37.0 + ] + ], + [ + [ + 33.75 + ] + ], + [ + [ + 85.0 + ] + ], + [ + [ + 62.25 + ] + ], + [ + [ + 46.0 + ] + ], + [ + [ + 53.0 + ] + ], + [ + [ + 81.5 + ] + ], + [ + [ + 75.5 + ] + ], + [ + [ + 67.0 + ] + ], + [ + [ + 50.0 + ] + ], + [ + [ + 56.5 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 91.5 + ] + ] + ] + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7109375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.310546875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11181640625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2490234375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.32421875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 23.75 + ] + ] + ], + "outputs": [ + [ + [ + 12.375 + ] + ], + [ + [ + 7.393830726553595e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.72265625 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 53.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.0 + ] + ] + ], + "outputs": [ + [ + [ + 12.8125 + ] + ], + [ + [ + 22.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8046875 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.9609375 + ] + ], + [ + [ + 16.375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.9375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.9609375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.0 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.9375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.625 + ] + ], + [ + [ + 16.375 + ] + ], + [ + [ + 6.78125 + ] + ] + ], + "outputs": [ + [ + [ + 4.96875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.20.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 42.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.365234375 + ] + ] + } + }, + "language_model.model.layers.20.feed_forward.down_proj": { + "inputs": [ + [ + [ + 15.6875 + ] + ] + ], + "outputs": [ + [ + [ + 10.5 + ] + ], + [ + [ + 41.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.443359375 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 58.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.3125 + ] + ] + ], + "outputs": [ + [ + [ + 26.0 + ] + ], + [ + [ + 35.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.431640625 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.3125 + ] + ], + [ + [ + 15.3125 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.9375 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.3125 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.28125 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.3125 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.9375 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.875 + ] + ], + [ + [ + 15.25 + ] + ], + [ + [ + 8.9375 + ] + ] + ], + "outputs": [ + [ + [ + 3.796875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.21.feed_forward.router": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.45703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "outputs": [ + [ + [ + 7.25 + ] + ], + [ + [ + 77.5 + ] + ], + [ + [ + 33.5 + ] + ], + [ + [ + 66.5 + ] + ], + [ + [ + 81.0 + ] + ], + [ + [ + 72.0 + ] + ], + [ + [ + 102.0 + ] + ], + [ + [ + 63.5 + ] + ], + [ + [ + 98.0 + ] + ], + [ + [ + 48.75 + ] + ], + [ + [ + 72.5 + ] + ], + [ + [ + 55.75 + ] + ], + [ + [ + 57.75 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 59.75 + ] + ], + [ + [ + 97.0 + ] + ], + [ + [ + 63.25 + ] + ] + ] + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34765625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 48.75 + ] + ] + ], + "outputs": [ + [ + [ + 8.625 + ] + ], + [ + [ + 1.022882481131361e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.47265625 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 57.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.6875 + ] + ] + ], + "outputs": [ + [ + [ + 15.75 + ] + ], + [ + [ + 28.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.328125 + ] + ], + [ + [ + 16.0 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.71875 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.328125 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.4375 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.0 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.71875 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.6875 + ] + ], + [ + [ + 16.0 + ] + ], + [ + [ + 6.71875 + ] + ] + ], + "outputs": [ + [ + [ + 5.6875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.22.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 46.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.38671875 + ] + ] + } + }, + "language_model.model.layers.22.feed_forward.down_proj": { + "inputs": [ + [ + [ + 15.3125 + ] + ] + ], + "outputs": [ + [ + [ + 14.875 + ] + ], + [ + [ + 77.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.60546875 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 69.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.46875 + ] + ] + ], + "outputs": [ + [ + [ + 70.0 + ] + ], + [ + [ + 79.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.125 + ] + ], + [ + [ + 17.875 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.625 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.125 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.71875 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.875 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.625 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.9375 + ] + ], + [ + [ + 17.875 + ] + ], + [ + [ + 11.625 + ] + ] + ], + "outputs": [ + [ + [ + 7.4375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.23.feed_forward.router": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.474609375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 235.0 + ] + ], + [ + [ + 125.0 + ] + ], + [ + [ + 126.5 + ] + ], + [ + [ + 45.0 + ] + ], + [ + [ + 59.25 + ] + ], + [ + [ + 73.5 + ] + ], + [ + [ + 25.625 + ] + ], + [ + [ + 45.75 + ] + ], + [ + [ + 71.5 + ] + ], + [ + [ + 244.0 + ] + ], + [ + [ + 143.0 + ] + ], + [ + [ + 89.5 + ] + ], + [ + [ + 138.0 + ] + ], + [ + [ + 89.0 + ] + ], + [ + [ + 71.0 + ] + ], + [ + [ + 36.75 + ] + ], + [ + [ + 58.0 + ] + ] + ] + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10693359375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.48046875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4296875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10498046875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6953125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.392578125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.369140625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 35.5 + ] + ] + ], + "outputs": [ + [ + [ + 6.4375 + ] + ], + [ + [ + 5.0469125464958525e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.34375 + ] + ] + ], + "outputs": [ + [ + [ + 48.0 + ] + ], + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.95703125 + ] + ], + [ + [ + 17.25 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.25 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.95703125 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.6875 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.25 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 12.25 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.625 + ] + ], + [ + [ + 17.25 + ] + ], + [ + [ + 12.25 + ] + ] + ], + "outputs": [ + [ + [ + 6.34375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.24.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.24.feed_forward.down_proj": { + "inputs": [ + [ + [ + 59.25 + ] + ] + ], + "outputs": [ + [ + [ + 21.75 + ] + ], + [ + [ + 100.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.59375 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.71875 + ] + ] + ], + "outputs": [ + [ + [ + 53.0 + ] + ], + [ + [ + 69.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0703125 + ] + ], + [ + [ + 24.875 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.5625 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0703125 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.71875 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 24.875 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.5625 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.625 + ] + ], + [ + [ + 24.875 + ] + ], + [ + [ + 6.5625 + ] + ] + ], + "outputs": [ + [ + [ + 5.28125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.25.feed_forward.router": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.412109375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "outputs": [ + [ + [ + 13.4375 + ] + ], + [ + [ + 88.0 + ] + ], + [ + [ + 61.25 + ] + ], + [ + [ + 75.0 + ] + ], + [ + [ + 102.5 + ] + ], + [ + [ + 56.5 + ] + ], + [ + [ + 40.0 + ] + ], + [ + [ + 43.5 + ] + ], + [ + [ + 32.75 + ] + ], + [ + [ + 78.5 + ] + ], + [ + [ + 42.5 + ] + ], + [ + [ + 78.5 + ] + ], + [ + [ + 45.5 + ] + ], + [ + [ + 37.75 + ] + ], + [ + [ + 103.5 + ] + ], + [ + [ + 83.0 + ] + ], + [ + [ + 62.0 + ] + ] + ] + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11376953125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33984375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 24.375 + ] + ] + ], + "outputs": [ + [ + [ + 9.5 + ] + ], + [ + [ + 7.393830726553595e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.51171875 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 63.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.59375 + ] + ] + ], + "outputs": [ + [ + [ + 30.875 + ] + ], + [ + [ + 48.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.765625 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.21875 + ] + ], + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.4375 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.21875 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.0625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.4375 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.75 + ] + ], + [ + [ + 17.5 + ] + ], + [ + [ + 8.4375 + ] + ] + ], + "outputs": [ + [ + [ + 6.59375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.26.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 45.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.26.feed_forward.down_proj": { + "inputs": [ + [ + [ + 28.875 + ] + ] + ], + "outputs": [ + [ + [ + 22.125 + ] + ], + [ + [ + 113.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.78515625 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.8125 + ] + ] + ], + "outputs": [ + [ + [ + 60.25 + ] + ], + [ + [ + 71.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.73046875 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.87890625 + ] + ], + [ + [ + 11.375 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.875 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.87890625 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.21875 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.875 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.125 + ] + ], + [ + [ + 11.375 + ] + ], + [ + [ + 5.875 + ] + ] + ], + "outputs": [ + [ + [ + 4.8125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.27.feed_forward.router": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "outputs": [ + [ + [ + 14.4375 + ] + ], + [ + [ + 67.5 + ] + ], + [ + [ + 45.75 + ] + ], + [ + [ + 185.0 + ] + ], + [ + [ + 75.5 + ] + ], + [ + [ + 56.0 + ] + ], + [ + [ + 54.25 + ] + ], + [ + [ + 61.25 + ] + ], + [ + [ + 60.75 + ] + ], + [ + [ + 50.25 + ] + ], + [ + [ + 75.5 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 32.0 + ] + ], + [ + [ + 108.5 + ] + ], + [ + [ + 84.5 + ] + ], + [ + [ + 79.0 + ] + ], + [ + [ + 50.25 + ] + ] + ] + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.48828125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10888671875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.283203125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.357421875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 38.5 + ] + ] + ], + "outputs": [ + [ + [ + 9.375 + ] + ], + [ + [ + 7.393830726553595e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5859375 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 78.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.o_proj": { + "inputs": [ + [ + [ + 10.375 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 97.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.328125 + ] + ], + [ + [ + 18.75 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.6875 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.328125 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.375 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.75 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.6875 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.625 + ] + ], + [ + [ + 18.75 + ] + ], + [ + [ + 10.6875 + ] + ] + ], + "outputs": [ + [ + [ + 10.375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.28.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5390625 + ] + ] + } + }, + "language_model.model.layers.28.feed_forward.down_proj": { + "inputs": [ + [ + [ + 177.0 + ] + ] + ], + "outputs": [ + [ + [ + 140.0 + ] + ], + [ + [ + 149.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.78125 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 67.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.o_proj": { + "inputs": [ + [ + [ + 11.3125 + ] + ] + ], + "outputs": [ + [ + [ + 78.0 + ] + ], + [ + [ + 84.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.015625 + ] + ], + [ + [ + 20.875 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.125 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.015625 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.75 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.875 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.125 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.25 + ] + ], + [ + [ + 20.875 + ] + ], + [ + [ + 12.8125 + ] + ] + ], + "outputs": [ + [ + [ + 11.3125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.29.feed_forward.router": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.333984375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "outputs": [ + [ + [ + 13.625 + ] + ], + [ + [ + 48.0 + ] + ], + [ + [ + 32.0 + ] + ], + [ + [ + 43.5 + ] + ], + [ + [ + 53.75 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 68.0 + ] + ], + [ + [ + 106.0 + ] + ], + [ + [ + 84.0 + ] + ], + [ + [ + 78.5 + ] + ], + [ + [ + 160.0 + ] + ], + [ + [ + 43.75 + ] + ], + [ + [ + 102.0 + ] + ], + [ + [ + 178.0 + ] + ], + [ + [ + 32.5 + ] + ], + [ + [ + 40.5 + ] + ], + [ + [ + 125.0 + ] + ] + ] + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37109375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.44921875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 34.75 + ] + ] + ], + "outputs": [ + [ + [ + 11.6875 + ] + ], + [ + [ + 8.639981972601953e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.55859375 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.46875 + ] + ] + ], + "outputs": [ + [ + [ + 50.25 + ] + ], + [ + [ + 56.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0546875 + ] + ], + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.28125 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0546875 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.21875 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.28125 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.0 + ] + ], + [ + [ + 18.625 + ] + ], + [ + [ + 7.28125 + ] + ] + ], + "outputs": [ + [ + [ + 6.46875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.30.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 32.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.515625 + ] + ] + } + }, + "language_model.model.layers.30.feed_forward.down_proj": { + "inputs": [ + [ + [ + 34.0 + ] + ] + ], + "outputs": [ + [ + [ + 30.25 + ] + ], + [ + [ + 159.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5234375 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.71875 + ] + ] + ], + "outputs": [ + [ + [ + 77.0 + ] + ], + [ + [ + 115.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.984375 + ] + ], + [ + [ + 21.0 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.1875 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.984375 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.40625 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 21.0 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 12.1875 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.5 + ] + ], + [ + [ + 21.0 + ] + ], + [ + [ + 12.0 + ] + ] + ], + "outputs": [ + [ + [ + 7.71875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.31.feed_forward.router": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "outputs": [ + [ + [ + 23.25 + ] + ], + [ + [ + 84.5 + ] + ], + [ + [ + 167.0 + ] + ], + [ + [ + 73.5 + ] + ], + [ + [ + 76.0 + ] + ], + [ + [ + 115.0 + ] + ], + [ + [ + 98.0 + ] + ], + [ + [ + 93.5 + ] + ], + [ + [ + 50.25 + ] + ], + [ + [ + 73.0 + ] + ], + [ + [ + 98.0 + ] + ], + [ + [ + 69.5 + ] + ], + [ + [ + 64.5 + ] + ], + [ + [ + 70.5 + ] + ], + [ + [ + 52.0 + ] + ], + [ + [ + 220.0 + ] + ], + [ + [ + 76.5 + ] + ] + ] + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3203125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.240234375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.240234375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.53515625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 86.0 + ] + ] + ], + "outputs": [ + [ + [ + 40.0 + ] + ], + [ + [ + 8.639981972601953e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.90625 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 53.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.375 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 109.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.78125 + ] + ], + [ + [ + 20.0 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.1875 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.78125 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.34375 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.0 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.1875 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 20.25 + ] + ], + [ + [ + 20.0 + ] + ], + [ + [ + 7.59375 + ] + ] + ], + "outputs": [ + [ + [ + 7.125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.32.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 32.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.32.feed_forward.down_proj": { + "inputs": [ + [ + [ + 42.0 + ] + ] + ], + "outputs": [ + [ + [ + 29.375 + ] + ], + [ + [ + 75.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.83203125 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 44.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.5625 + ] + ] + ], + "outputs": [ + [ + [ + 94.0 + ] + ], + [ + [ + 132.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1875 + ] + ], + [ + [ + 21.0 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.8125 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1875 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.46875 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 21.0 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.8125 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.75 + ] + ], + [ + [ + 21.0 + ] + ], + [ + [ + 8.8125 + ] + ] + ], + "outputs": [ + [ + [ + 8.5625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.33.feed_forward.router": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "outputs": [ + [ + [ + 35.75 + ] + ], + [ + [ + 104.0 + ] + ], + [ + [ + 126.0 + ] + ], + [ + [ + 123.5 + ] + ], + [ + [ + 48.5 + ] + ], + [ + [ + 91.0 + ] + ], + [ + [ + 123.0 + ] + ], + [ + [ + 153.0 + ] + ], + [ + [ + 122.0 + ] + ], + [ + [ + 102.5 + ] + ], + [ + [ + 59.0 + ] + ], + [ + [ + 102.5 + ] + ], + [ + [ + 138.0 + ] + ], + [ + [ + 62.75 + ] + ], + [ + [ + 96.0 + ] + ], + [ + [ + 74.5 + ] + ], + [ + [ + 109.0 + ] + ] + ] + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.244140625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28515625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2490234375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.240234375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.443359375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 69.5 + ] + ] + ], + "outputs": [ + [ + [ + 49.75 + ] + ], + [ + [ + 8.639981972601953e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8671875 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 54.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.59375 + ] + ] + ], + "outputs": [ + [ + [ + 55.75 + ] + ], + [ + [ + 98.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1953125 + ] + ], + [ + [ + 19.125 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.125 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1953125 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.59375 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.125 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.125 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.3125 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 8.125 + ] + ] + ], + "outputs": [ + [ + [ + 7.4375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.34.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.41015625 + ] + ] + } + }, + "language_model.model.layers.34.feed_forward.down_proj": { + "inputs": [ + [ + [ + 70.0 + ] + ] + ], + "outputs": [ + [ + [ + 36.5 + ] + ], + [ + [ + 162.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.470703125 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 54.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.4375 + ] + ] + ], + "outputs": [ + [ + [ + 110.5 + ] + ], + [ + [ + 116.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.85546875 + ] + ], + [ + [ + 15.0625 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.5 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.85546875 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.46875 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.0625 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.5 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.625 + ] + ], + [ + [ + 15.0625 + ] + ], + [ + [ + 10.5 + ] + ] + ], + "outputs": [ + [ + [ + 8.4375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.35.feed_forward.router": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "outputs": [ + [ + [ + 21.125 + ] + ], + [ + [ + 137.0 + ] + ], + [ + [ + 110.5 + ] + ], + [ + [ + 59.25 + ] + ], + [ + [ + 89.0 + ] + ], + [ + [ + 144.0 + ] + ], + [ + [ + 95.0 + ] + ], + [ + [ + 99.5 + ] + ], + [ + [ + 47.25 + ] + ], + [ + [ + 135.0 + ] + ], + [ + [ + 85.0 + ] + ], + [ + [ + 181.0 + ] + ], + [ + [ + 104.0 + ] + ], + [ + [ + 119.0 + ] + ], + [ + [ + 170.0 + ] + ], + [ + [ + 119.5 + ] + ], + [ + [ + 79.5 + ] + ] + ] + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28515625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.396484375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.361328125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4140625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.416015625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 102.0 + ] + ] + ], + "outputs": [ + [ + [ + 11.0 + ] + ], + [ + [ + 7.393830726553595e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.455078125 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 56.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.625 + ] + ] + ], + "outputs": [ + [ + [ + 36.75 + ] + ], + [ + [ + 57.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.53125 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.359375 + ] + ], + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.125 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.359375 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.15625 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.125 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.5 + ] + ], + [ + [ + 17.5 + ] + ], + [ + [ + 8.125 + ] + ] + ], + "outputs": [ + [ + [ + 7.625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.36.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 29.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.36.feed_forward.down_proj": { + "inputs": [ + [ + [ + 86.0 + ] + ] + ], + "outputs": [ + [ + [ + 43.0 + ] + ], + [ + [ + 94.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7578125 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.o_proj": { + "inputs": [ + [ + [ + 11.1875 + ] + ] + ], + "outputs": [ + [ + [ + 97.5 + ] + ], + [ + [ + 150.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 2.203125 + ] + ], + [ + [ + 14.875 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.75 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.203125 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.0 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.875 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.75 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 25.5 + ] + ], + [ + [ + 14.4375 + ] + ], + [ + [ + 13.75 + ] + ] + ], + "outputs": [ + [ + [ + 10.5625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.37.feed_forward.router": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "outputs": [ + [ + [ + 30.25 + ] + ], + [ + [ + 105.5 + ] + ], + [ + [ + 130.0 + ] + ], + [ + [ + 132.0 + ] + ], + [ + [ + 229.0 + ] + ], + [ + [ + 153.0 + ] + ], + [ + [ + 120.5 + ] + ], + [ + [ + 111.5 + ] + ], + [ + [ + 114.5 + ] + ], + [ + [ + 139.0 + ] + ], + [ + [ + 77.0 + ] + ], + [ + [ + 118.0 + ] + ], + [ + [ + 168.0 + ] + ], + [ + [ + 117.0 + ] + ], + [ + [ + 98.0 + ] + ], + [ + [ + 90.5 + ] + ], + [ + [ + 110.5 + ] + ] + ] + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.287109375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.421875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1025390625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2490234375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.42578125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.388671875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28515625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.322265625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.341796875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 103.0 + ] + ] + ], + "outputs": [ + [ + [ + 11.8125 + ] + ], + [ + [ + 7.393830726553595e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.58203125 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 47.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.4375 + ] + ] + ], + "outputs": [ + [ + [ + 29.75 + ] + ], + [ + [ + 66.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.515625 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.671875 + ] + ], + [ + [ + 19.125 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.1875 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.671875 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.0 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.125 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.1875 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 21.0 + ] + ], + [ + [ + 19.125 + ] + ], + [ + [ + 10.1875 + ] + ] + ], + "outputs": [ + [ + [ + 8.4375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.38.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 31.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.38.feed_forward.down_proj": { + "inputs": [ + [ + [ + 118.5 + ] + ] + ], + "outputs": [ + [ + [ + 31.125 + ] + ], + [ + [ + 262.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6015625 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 52.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.5 + ] + ] + ], + "outputs": [ + [ + [ + 126.5 + ] + ], + [ + [ + 154.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.9765625 + ] + ], + [ + [ + 14.3125 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.9375 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.9765625 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.75 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.3125 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.9375 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.375 + ] + ], + [ + [ + 14.3125 + ] + ], + [ + [ + 8.9375 + ] + ] + ], + "outputs": [ + [ + [ + 8.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.39.feed_forward.router": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "outputs": [ + [ + [ + 31.625 + ] + ], + [ + [ + 127.0 + ] + ], + [ + [ + 141.0 + ] + ], + [ + [ + 191.0 + ] + ], + [ + [ + 153.0 + ] + ], + [ + [ + 127.0 + ] + ], + [ + [ + 170.0 + ] + ], + [ + [ + 102.5 + ] + ], + [ + [ + 166.0 + ] + ], + [ + [ + 69.5 + ] + ], + [ + [ + 172.0 + ] + ], + [ + [ + 106.5 + ] + ], + [ + [ + 110.0 + ] + ], + [ + [ + 158.0 + ] + ], + [ + [ + 147.0 + ] + ], + [ + [ + 89.0 + ] + ], + [ + [ + 85.0 + ] + ] + ] + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.310546875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.515625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 115.0 + ] + ] + ], + "outputs": [ + [ + [ + 20.625 + ] + ], + [ + [ + 7.393830726553595e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.498046875 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 46.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.o_proj": { + "inputs": [ + [ + [ + 13.75 + ] + ] + ], + "outputs": [ + [ + [ + 94.0 + ] + ], + [ + [ + 102.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 2.234375 + ] + ], + [ + [ + 15.4375 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 16.25 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.234375 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.75 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.4375 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 16.25 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 26.125 + ] + ], + [ + [ + 15.4375 + ] + ], + [ + [ + 16.25 + ] + ] + ], + "outputs": [ + [ + [ + 13.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.40.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.40.feed_forward.down_proj": { + "inputs": [ + [ + [ + 172.0 + ] + ] + ], + "outputs": [ + [ + [ + 28.5 + ] + ], + [ + [ + 127.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6328125 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 35.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.o_proj": { + "inputs": [ + [ + [ + 12.5 + ] + ] + ], + "outputs": [ + [ + [ + 160.0 + ] + ], + [ + [ + 177.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.7265625 + ] + ], + [ + [ + 17.75 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.1875 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.7265625 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.6875 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.75 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.1875 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 20.625 + ] + ], + [ + [ + 17.75 + ] + ], + [ + [ + 13.1875 + ] + ] + ], + "outputs": [ + [ + [ + 12.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.41.feed_forward.router": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "outputs": [ + [ + [ + 53.75 + ] + ], + [ + [ + 204.0 + ] + ], + [ + [ + 304.0 + ] + ], + [ + [ + 200.0 + ] + ], + [ + [ + 151.0 + ] + ], + [ + [ + 142.0 + ] + ], + [ + [ + 189.0 + ] + ], + [ + [ + 118.5 + ] + ], + [ + [ + 276.0 + ] + ], + [ + [ + 226.0 + ] + ], + [ + [ + 144.0 + ] + ], + [ + [ + 150.0 + ] + ], + [ + [ + 110.5 + ] + ], + [ + [ + 204.0 + ] + ], + [ + [ + 159.0 + ] + ], + [ + [ + 175.0 + ] + ], + [ + [ + 148.0 + ] + ] + ] + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10693359375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1025390625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.32421875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.400390625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.388671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 201.0 + ] + ] + ], + "outputs": [ + [ + [ + 68.5 + ] + ], + [ + [ + 8.639981972601953e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 1.0234375 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 39.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.o_proj": { + "inputs": [ + [ + [ + 14.125 + ] + ] + ], + "outputs": [ + [ + [ + 65.0 + ] + ], + [ + [ + 146.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5703125 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.4453125 + ] + ], + [ + [ + 14.25 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 17.0 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4453125 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.8125 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.25 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 17.0 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 17.75 + ] + ], + [ + [ + 14.25 + ] + ], + [ + [ + 17.0 + ] + ] + ], + "outputs": [ + [ + [ + 14.125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.42.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 43.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.42.feed_forward.down_proj": { + "inputs": [ + [ + [ + 217.0 + ] + ] + ], + "outputs": [ + [ + [ + 70.5 + ] + ], + [ + [ + 218.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.71875 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 43.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.o_proj": { + "inputs": [ + [ + [ + 13.75 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 232.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.3125 + ] + ], + [ + [ + 15.625 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.3125 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.6875 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.625 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.8125 + ] + ], + [ + [ + 15.625 + ] + ], + [ + [ + 16.125 + ] + ] + ], + "outputs": [ + [ + [ + 13.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.43.feed_forward.router": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "outputs": [ + [ + [ + 52.25 + ] + ], + [ + [ + 157.0 + ] + ], + [ + [ + 178.0 + ] + ], + [ + [ + 118.5 + ] + ], + [ + [ + 179.0 + ] + ], + [ + [ + 227.0 + ] + ], + [ + [ + 133.0 + ] + ], + [ + [ + 210.0 + ] + ], + [ + [ + 173.0 + ] + ], + [ + [ + 42.75 + ] + ], + [ + [ + 92.0 + ] + ], + [ + [ + 99.0 + ] + ], + [ + [ + 206.0 + ] + ], + [ + [ + 118.0 + ] + ], + [ + [ + 127.0 + ] + ], + [ + [ + 97.5 + ] + ], + [ + [ + 260.0 + ] + ] + ] + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.244140625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10888671875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11767578125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09326171875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 178.0 + ] + ] + ], + "outputs": [ + [ + [ + 36.0 + ] + ], + [ + [ + 5.0469125464958525e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.59375 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 29.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.o_proj": { + "inputs": [ + [ + [ + 12.5 + ] + ] + ], + "outputs": [ + [ + [ + 81.0 + ] + ], + [ + [ + 170.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.6796875 + ] + ], + [ + [ + 25.5 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.5625 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.6796875 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.375 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 25.5 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 14.5625 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 20.75 + ] + ], + [ + [ + 25.5 + ] + ], + [ + [ + 14.5625 + ] + ] + ], + "outputs": [ + [ + [ + 12.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.44.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 42.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.44.feed_forward.down_proj": { + "inputs": [ + [ + [ + 224.0 + ] + ] + ], + "outputs": [ + [ + [ + 75.5 + ] + ], + [ + [ + 200.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.51171875 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 28.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.o_proj": { + "inputs": [ + [ + [ + 17.5 + ] + ] + ], + "outputs": [ + [ + [ + 170.0 + ] + ], + [ + [ + 258.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33984375 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.671875 + ] + ], + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 17.875 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.671875 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 16.125 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 17.875 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 20.375 + ] + ], + [ + [ + 18.125 + ] + ], + [ + [ + 17.875 + ] + ] + ], + "outputs": [ + [ + [ + 17.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.45.feed_forward.router": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "outputs": [ + [ + [ + 70.0 + ] + ], + [ + [ + 23.75 + ] + ], + [ + [ + 140.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 38.5 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 83.5 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 221.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ] + ] + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0869140625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09130859375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.423828125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08642578125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 416.0 + ] + ] + ], + "outputs": [ + [ + [ + 45.75 + ] + ], + [ + [ + 5.0469125464958525e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.44921875 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 27.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.o_proj": { + "inputs": [ + [ + [ + 19.75 + ] + ] + ], + "outputs": [ + [ + [ + 126.0 + ] + ], + [ + [ + 221.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.5234375 + ] + ], + [ + [ + 17.625 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 20.5 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.5234375 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 18.75 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.625 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 20.5 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 18.75 + ] + ], + [ + [ + 17.625 + ] + ], + [ + [ + 20.5 + ] + ] + ], + "outputs": [ + [ + [ + 19.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.46.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 40.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.46.feed_forward.down_proj": { + "inputs": [ + [ + [ + 980.0 + ] + ] + ], + "outputs": [ + [ + [ + 1680.0 + ] + ], + [ + [ + 2400.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.447265625 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.o_proj": { + "inputs": [ + [ + [ + 12.375 + ] + ] + ], + "outputs": [ + [ + [ + 96.5 + ] + ], + [ + [ + 125.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.53515625 + ] + ], + [ + [ + 16.125 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.1875 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.53515625 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.375 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.125 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.1875 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 6.625 + ] + ], + [ + [ + 16.125 + ] + ], + [ + [ + 13.1875 + ] + ] + ], + "outputs": [ + [ + [ + 12.375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.47.feed_forward.router": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "outputs": [ + [ + [ + 2208.0 + ] + ], + [ + [ + 29.375 + ] + ], + [ + [ + 213.0 + ] + ], + [ + [ + 358.0 + ] + ], + [ + [ + 792.0 + ] + ], + [ + [ + 232.0 + ] + ], + [ + [ + 73.0 + ] + ], + [ + [ + 72.5 + ] + ], + [ + [ + 131.0 + ] + ], + [ + [ + 143.0 + ] + ], + [ + [ + 79.0 + ] + ], + [ + [ + 104.5 + ] + ], + [ + [ + 256.0 + ] + ], + [ + [ + 215.0 + ] + ], + [ + [ + 121.0 + ] + ], + [ + [ + 888.0 + ] + ], + [ + [ + 342.0 + ] + ] + ] + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1005859375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1044921875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.470703125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.953125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7734375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.75 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7265625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.83203125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.310546875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.44921875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.35546875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8359375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.9140625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7265625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.412109375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 1648.0 + ] + ] + ], + "outputs": [ + [ + [ + 1280.0 + ] + ], + [ + [ + 8.639981972601953e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.435546875 + ] + ] + } + }, + "language_model.lm_head": { + "inputs": [ + [ + [ + 54.25 + ] + ] + ], + "params": { + "weight": [ + [ + 1.0078125 + ] + ] + } + } + } +} \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_3_8.npz b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_3_8.npz new file mode 100644 index 000000000000..910e37927d6b Binary files /dev/null and b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_3_8.npz differ diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_3_8_mod_list.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_3_8_mod_list.json new file mode 100644 index 000000000000..449fce78b77d --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_3_8_mod_list.json @@ -0,0 +1,1559 @@ +[ + "vision_model.patch_embedding.linear", + "vision_model.model.layers.0.self_attn.qkv_proj", + "vision_model.model.layers.0.self_attn.o_proj", + "vision_model.model.layers.0.mlp.fc1", + "vision_model.model.layers.0.mlp.fc2", + "vision_model.model.layers.1.self_attn.qkv_proj", + "vision_model.model.layers.1.self_attn.o_proj", + "vision_model.model.layers.1.mlp.fc1", + "vision_model.model.layers.1.mlp.fc2", + "vision_model.model.layers.2.self_attn.qkv_proj", + "vision_model.model.layers.2.self_attn.o_proj", + "vision_model.model.layers.2.mlp.fc1", + "vision_model.model.layers.2.mlp.fc2", + "vision_model.model.layers.3.self_attn.qkv_proj", + "vision_model.model.layers.3.self_attn.o_proj", + "vision_model.model.layers.3.mlp.fc1", + "vision_model.model.layers.3.mlp.fc2", + "vision_model.model.layers.4.self_attn.qkv_proj", + "vision_model.model.layers.4.self_attn.o_proj", + "vision_model.model.layers.4.mlp.fc1", + "vision_model.model.layers.4.mlp.fc2", + "vision_model.model.layers.5.self_attn.qkv_proj", + "vision_model.model.layers.5.self_attn.o_proj", + "vision_model.model.layers.5.mlp.fc1", + "vision_model.model.layers.5.mlp.fc2", + "vision_model.model.layers.6.self_attn.qkv_proj", + "vision_model.model.layers.6.self_attn.o_proj", + "vision_model.model.layers.6.mlp.fc1", + "vision_model.model.layers.6.mlp.fc2", + "vision_model.model.layers.7.self_attn.qkv_proj", + "vision_model.model.layers.7.self_attn.o_proj", + "vision_model.model.layers.7.mlp.fc1", + "vision_model.model.layers.7.mlp.fc2", + "vision_model.model.layers.8.self_attn.qkv_proj", + "vision_model.model.layers.8.self_attn.o_proj", + "vision_model.model.layers.8.mlp.fc1", + "vision_model.model.layers.8.mlp.fc2", + "vision_model.model.layers.9.self_attn.qkv_proj", + "vision_model.model.layers.9.self_attn.o_proj", + "vision_model.model.layers.9.mlp.fc1", + "vision_model.model.layers.9.mlp.fc2", + "vision_model.model.layers.10.self_attn.qkv_proj", + "vision_model.model.layers.10.self_attn.o_proj", + "vision_model.model.layers.10.mlp.fc1", + "vision_model.model.layers.10.mlp.fc2", + "vision_model.model.layers.11.self_attn.qkv_proj", + "vision_model.model.layers.11.self_attn.o_proj", + "vision_model.model.layers.11.mlp.fc1", + "vision_model.model.layers.11.mlp.fc2", + "vision_model.model.layers.12.self_attn.qkv_proj", + "vision_model.model.layers.12.self_attn.o_proj", + "vision_model.model.layers.12.mlp.fc1", + "vision_model.model.layers.12.mlp.fc2", + "vision_model.model.layers.13.self_attn.qkv_proj", + "vision_model.model.layers.13.self_attn.o_proj", + "vision_model.model.layers.13.mlp.fc1", + "vision_model.model.layers.13.mlp.fc2", + "vision_model.model.layers.14.self_attn.qkv_proj", + "vision_model.model.layers.14.self_attn.o_proj", + "vision_model.model.layers.14.mlp.fc1", + "vision_model.model.layers.14.mlp.fc2", + "vision_model.model.layers.15.self_attn.qkv_proj", + "vision_model.model.layers.15.self_attn.o_proj", + "vision_model.model.layers.15.mlp.fc1", + "vision_model.model.layers.15.mlp.fc2", + "vision_model.model.layers.16.self_attn.qkv_proj", + "vision_model.model.layers.16.self_attn.o_proj", + "vision_model.model.layers.16.mlp.fc1", + "vision_model.model.layers.16.mlp.fc2", + "vision_model.model.layers.17.self_attn.qkv_proj", + "vision_model.model.layers.17.self_attn.o_proj", + "vision_model.model.layers.17.mlp.fc1", + "vision_model.model.layers.17.mlp.fc2", + "vision_model.model.layers.18.self_attn.qkv_proj", + "vision_model.model.layers.18.self_attn.o_proj", + "vision_model.model.layers.18.mlp.fc1", + "vision_model.model.layers.18.mlp.fc2", + "vision_model.model.layers.19.self_attn.qkv_proj", + "vision_model.model.layers.19.self_attn.o_proj", + "vision_model.model.layers.19.mlp.fc1", + "vision_model.model.layers.19.mlp.fc2", + "vision_model.model.layers.20.self_attn.qkv_proj", + "vision_model.model.layers.20.self_attn.o_proj", + "vision_model.model.layers.20.mlp.fc1", + "vision_model.model.layers.20.mlp.fc2", + "vision_model.model.layers.21.self_attn.qkv_proj", + "vision_model.model.layers.21.self_attn.o_proj", + "vision_model.model.layers.21.mlp.fc1", + "vision_model.model.layers.21.mlp.fc2", + "vision_model.model.layers.22.self_attn.qkv_proj", + "vision_model.model.layers.22.self_attn.o_proj", + "vision_model.model.layers.22.mlp.fc1", + "vision_model.model.layers.22.mlp.fc2", + "vision_model.model.layers.23.self_attn.qkv_proj", + "vision_model.model.layers.23.self_attn.o_proj", + "vision_model.model.layers.23.mlp.fc1", + "vision_model.model.layers.23.mlp.fc2", + "vision_model.model.layers.24.self_attn.qkv_proj", + "vision_model.model.layers.24.self_attn.o_proj", + "vision_model.model.layers.24.mlp.fc1", + "vision_model.model.layers.24.mlp.fc2", + "vision_model.model.layers.25.self_attn.qkv_proj", + "vision_model.model.layers.25.self_attn.o_proj", + "vision_model.model.layers.25.mlp.fc1", + "vision_model.model.layers.25.mlp.fc2", + "vision_model.model.layers.26.self_attn.qkv_proj", + "vision_model.model.layers.26.self_attn.o_proj", + "vision_model.model.layers.26.mlp.fc1", + "vision_model.model.layers.26.mlp.fc2", + "vision_model.model.layers.27.self_attn.qkv_proj", + "vision_model.model.layers.27.self_attn.o_proj", + "vision_model.model.layers.27.mlp.fc1", + "vision_model.model.layers.27.mlp.fc2", + "vision_model.model.layers.28.self_attn.qkv_proj", + "vision_model.model.layers.28.self_attn.o_proj", + "vision_model.model.layers.28.mlp.fc1", + "vision_model.model.layers.28.mlp.fc2", + "vision_model.model.layers.29.self_attn.qkv_proj", + "vision_model.model.layers.29.self_attn.o_proj", + "vision_model.model.layers.29.mlp.fc1", + "vision_model.model.layers.29.mlp.fc2", + "vision_model.model.layers.30.self_attn.qkv_proj", + "vision_model.model.layers.30.self_attn.o_proj", + "vision_model.model.layers.30.mlp.fc1", + "vision_model.model.layers.30.mlp.fc2", + "vision_model.model.layers.31.self_attn.qkv_proj", + "vision_model.model.layers.31.self_attn.o_proj", + "vision_model.model.layers.31.mlp.fc1", + "vision_model.model.layers.31.mlp.fc2", + "vision_model.model.layers.32.self_attn.qkv_proj", + "vision_model.model.layers.32.self_attn.o_proj", + "vision_model.model.layers.32.mlp.fc1", + "vision_model.model.layers.32.mlp.fc2", + "vision_model.model.layers.33.self_attn.qkv_proj", + "vision_model.model.layers.33.self_attn.o_proj", + "vision_model.model.layers.33.mlp.fc1", + "vision_model.model.layers.33.mlp.fc2", + "vision_model.vision_adapter.mlp.fc1", + "vision_model.vision_adapter.mlp.fc2", + "multi_modal_projector.linear_1", + "language_model.model.layers.0.self_attn.qkv_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.0.self_attn.attn.impl.softmax", + "language_model.model.layers.0.self_attn.attn.impl.matmul_av", + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.0.self_attn.attn.impl.k_cache", + "language_model.model.layers.0.self_attn.attn.impl.v_cache", + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.0.feed_forward.gate_up_proj", + "language_model.model.layers.0.feed_forward.down_proj", + "language_model.model.layers.1.self_attn.qkv_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.1.self_attn.attn.impl.softmax", + "language_model.model.layers.1.self_attn.attn.impl.matmul_av", + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.1.self_attn.attn.impl.k_cache", + "language_model.model.layers.1.self_attn.attn.impl.v_cache", + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.1.feed_forward.router", + "language_model.model.layers.1.feed_forward.experts", + "language_model.model.layers.1.feed_forward.experts.moe_op", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.1.feed_forward.shared_expert.down_proj", + "language_model.model.layers.2.self_attn.qkv_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.2.self_attn.attn.impl.softmax", + "language_model.model.layers.2.self_attn.attn.impl.matmul_av", + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.2.self_attn.attn.impl.k_cache", + "language_model.model.layers.2.self_attn.attn.impl.v_cache", + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.2.feed_forward.gate_up_proj", + "language_model.model.layers.2.feed_forward.down_proj", + "language_model.model.layers.3.self_attn.qkv_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.3.self_attn.attn.impl.softmax", + "language_model.model.layers.3.self_attn.attn.impl.matmul_av", + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.3.self_attn.attn.impl.k_cache", + "language_model.model.layers.3.self_attn.attn.impl.v_cache", + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.3.feed_forward.router", + "language_model.model.layers.3.feed_forward.experts", + "language_model.model.layers.3.feed_forward.experts.moe_op", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.3.feed_forward.shared_expert.down_proj", + "language_model.model.layers.4.self_attn.qkv_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.4.self_attn.attn.impl.softmax", + "language_model.model.layers.4.self_attn.attn.impl.matmul_av", + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.4.self_attn.attn.impl.k_cache", + "language_model.model.layers.4.self_attn.attn.impl.v_cache", + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.4.feed_forward.gate_up_proj", + "language_model.model.layers.4.feed_forward.down_proj", + "language_model.model.layers.5.self_attn.qkv_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.5.self_attn.attn.impl.softmax", + "language_model.model.layers.5.self_attn.attn.impl.matmul_av", + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.5.self_attn.attn.impl.k_cache", + "language_model.model.layers.5.self_attn.attn.impl.v_cache", + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.5.feed_forward.router", + "language_model.model.layers.5.feed_forward.experts", + "language_model.model.layers.5.feed_forward.experts.moe_op", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.5.feed_forward.shared_expert.down_proj", + "language_model.model.layers.6.self_attn.qkv_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.6.self_attn.attn.impl.softmax", + "language_model.model.layers.6.self_attn.attn.impl.matmul_av", + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.6.self_attn.attn.impl.k_cache", + "language_model.model.layers.6.self_attn.attn.impl.v_cache", + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.6.feed_forward.gate_up_proj", + "language_model.model.layers.6.feed_forward.down_proj", + "language_model.model.layers.7.self_attn.qkv_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.7.self_attn.attn.impl.softmax", + "language_model.model.layers.7.self_attn.attn.impl.matmul_av", + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.7.self_attn.attn.impl.k_cache", + "language_model.model.layers.7.self_attn.attn.impl.v_cache", + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.7.feed_forward.router", + "language_model.model.layers.7.feed_forward.experts", + "language_model.model.layers.7.feed_forward.experts.moe_op", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.7.feed_forward.shared_expert.down_proj", + "language_model.model.layers.8.self_attn.qkv_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.8.self_attn.attn.impl.softmax", + "language_model.model.layers.8.self_attn.attn.impl.matmul_av", + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.8.self_attn.attn.impl.k_cache", + "language_model.model.layers.8.self_attn.attn.impl.v_cache", + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.8.feed_forward.gate_up_proj", + "language_model.model.layers.8.feed_forward.down_proj", + "language_model.model.layers.9.self_attn.qkv_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.9.self_attn.attn.impl.softmax", + "language_model.model.layers.9.self_attn.attn.impl.matmul_av", + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.9.self_attn.attn.impl.k_cache", + "language_model.model.layers.9.self_attn.attn.impl.v_cache", + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.9.feed_forward.router", + "language_model.model.layers.9.feed_forward.experts", + "language_model.model.layers.9.feed_forward.experts.moe_op", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.9.feed_forward.shared_expert.down_proj", + "language_model.model.layers.10.self_attn.qkv_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.10.self_attn.attn.impl.softmax", + "language_model.model.layers.10.self_attn.attn.impl.matmul_av", + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.10.self_attn.attn.impl.k_cache", + "language_model.model.layers.10.self_attn.attn.impl.v_cache", + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.10.feed_forward.gate_up_proj", + "language_model.model.layers.10.feed_forward.down_proj", + "language_model.model.layers.11.self_attn.qkv_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.11.self_attn.attn.impl.softmax", + "language_model.model.layers.11.self_attn.attn.impl.matmul_av", + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.11.self_attn.attn.impl.k_cache", + "language_model.model.layers.11.self_attn.attn.impl.v_cache", + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.11.feed_forward.router", + "language_model.model.layers.11.feed_forward.experts", + "language_model.model.layers.11.feed_forward.experts.moe_op", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.11.feed_forward.shared_expert.down_proj", + "language_model.model.layers.12.self_attn.qkv_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.12.self_attn.attn.impl.softmax", + "language_model.model.layers.12.self_attn.attn.impl.matmul_av", + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.12.self_attn.attn.impl.k_cache", + "language_model.model.layers.12.self_attn.attn.impl.v_cache", + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.12.feed_forward.gate_up_proj", + "language_model.model.layers.12.feed_forward.down_proj", + "language_model.model.layers.13.self_attn.qkv_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.13.self_attn.attn.impl.softmax", + "language_model.model.layers.13.self_attn.attn.impl.matmul_av", + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.13.self_attn.attn.impl.k_cache", + "language_model.model.layers.13.self_attn.attn.impl.v_cache", + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.13.feed_forward.router", + "language_model.model.layers.13.feed_forward.experts", + "language_model.model.layers.13.feed_forward.experts.moe_op", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.13.feed_forward.shared_expert.down_proj", + "language_model.model.layers.14.self_attn.qkv_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.14.self_attn.attn.impl.softmax", + "language_model.model.layers.14.self_attn.attn.impl.matmul_av", + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.14.self_attn.attn.impl.k_cache", + "language_model.model.layers.14.self_attn.attn.impl.v_cache", + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.14.feed_forward.gate_up_proj", + "language_model.model.layers.14.feed_forward.down_proj", + "language_model.model.layers.15.self_attn.qkv_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.15.self_attn.attn.impl.softmax", + "language_model.model.layers.15.self_attn.attn.impl.matmul_av", + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.15.self_attn.attn.impl.k_cache", + "language_model.model.layers.15.self_attn.attn.impl.v_cache", + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.15.feed_forward.router", + "language_model.model.layers.15.feed_forward.experts", + "language_model.model.layers.15.feed_forward.experts.moe_op", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.15.feed_forward.shared_expert.down_proj", + "language_model.model.layers.16.self_attn.qkv_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.16.self_attn.attn.impl.softmax", + "language_model.model.layers.16.self_attn.attn.impl.matmul_av", + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.16.self_attn.attn.impl.k_cache", + "language_model.model.layers.16.self_attn.attn.impl.v_cache", + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.16.feed_forward.gate_up_proj", + "language_model.model.layers.16.feed_forward.down_proj", + "language_model.model.layers.17.self_attn.qkv_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.17.self_attn.attn.impl.softmax", + "language_model.model.layers.17.self_attn.attn.impl.matmul_av", + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.17.self_attn.attn.impl.k_cache", + "language_model.model.layers.17.self_attn.attn.impl.v_cache", + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.17.feed_forward.router", + "language_model.model.layers.17.feed_forward.experts", + "language_model.model.layers.17.feed_forward.experts.moe_op", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.17.feed_forward.shared_expert.down_proj", + "language_model.model.layers.18.self_attn.qkv_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.18.self_attn.attn.impl.softmax", + "language_model.model.layers.18.self_attn.attn.impl.matmul_av", + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.18.self_attn.attn.impl.k_cache", + "language_model.model.layers.18.self_attn.attn.impl.v_cache", + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.18.feed_forward.gate_up_proj", + "language_model.model.layers.18.feed_forward.down_proj", + "language_model.model.layers.19.self_attn.qkv_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.19.self_attn.attn.impl.softmax", + "language_model.model.layers.19.self_attn.attn.impl.matmul_av", + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.19.self_attn.attn.impl.k_cache", + "language_model.model.layers.19.self_attn.attn.impl.v_cache", + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.19.feed_forward.router", + "language_model.model.layers.19.feed_forward.experts", + "language_model.model.layers.19.feed_forward.experts.moe_op", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.19.feed_forward.shared_expert.down_proj", + "language_model.model.layers.20.self_attn.qkv_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.20.self_attn.attn.impl.softmax", + "language_model.model.layers.20.self_attn.attn.impl.matmul_av", + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.20.self_attn.attn.impl.k_cache", + "language_model.model.layers.20.self_attn.attn.impl.v_cache", + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.20.feed_forward.gate_up_proj", + "language_model.model.layers.20.feed_forward.down_proj", + "language_model.model.layers.21.self_attn.qkv_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.21.self_attn.attn.impl.softmax", + "language_model.model.layers.21.self_attn.attn.impl.matmul_av", + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.21.self_attn.attn.impl.k_cache", + "language_model.model.layers.21.self_attn.attn.impl.v_cache", + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.21.feed_forward.router", + "language_model.model.layers.21.feed_forward.experts", + "language_model.model.layers.21.feed_forward.experts.moe_op", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.21.feed_forward.shared_expert.down_proj", + "language_model.model.layers.22.self_attn.qkv_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.22.self_attn.attn.impl.softmax", + "language_model.model.layers.22.self_attn.attn.impl.matmul_av", + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.22.self_attn.attn.impl.k_cache", + "language_model.model.layers.22.self_attn.attn.impl.v_cache", + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.22.feed_forward.gate_up_proj", + "language_model.model.layers.22.feed_forward.down_proj", + "language_model.model.layers.23.self_attn.qkv_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.23.self_attn.attn.impl.softmax", + "language_model.model.layers.23.self_attn.attn.impl.matmul_av", + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.23.self_attn.attn.impl.k_cache", + "language_model.model.layers.23.self_attn.attn.impl.v_cache", + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.23.feed_forward.router", + "language_model.model.layers.23.feed_forward.experts", + "language_model.model.layers.23.feed_forward.experts.moe_op", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.23.feed_forward.shared_expert.down_proj", + "language_model.model.layers.24.self_attn.qkv_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.24.self_attn.attn.impl.softmax", + "language_model.model.layers.24.self_attn.attn.impl.matmul_av", + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.24.self_attn.attn.impl.k_cache", + "language_model.model.layers.24.self_attn.attn.impl.v_cache", + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.24.feed_forward.gate_up_proj", + "language_model.model.layers.24.feed_forward.down_proj", + "language_model.model.layers.25.self_attn.qkv_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.25.self_attn.attn.impl.softmax", + "language_model.model.layers.25.self_attn.attn.impl.matmul_av", + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.25.self_attn.attn.impl.k_cache", + "language_model.model.layers.25.self_attn.attn.impl.v_cache", + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.25.feed_forward.router", + "language_model.model.layers.25.feed_forward.experts", + "language_model.model.layers.25.feed_forward.experts.moe_op", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.25.feed_forward.shared_expert.down_proj", + "language_model.model.layers.26.self_attn.qkv_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.26.self_attn.attn.impl.softmax", + "language_model.model.layers.26.self_attn.attn.impl.matmul_av", + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.26.self_attn.attn.impl.k_cache", + "language_model.model.layers.26.self_attn.attn.impl.v_cache", + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.26.feed_forward.gate_up_proj", + "language_model.model.layers.26.feed_forward.down_proj", + "language_model.model.layers.27.self_attn.qkv_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.27.self_attn.attn.impl.softmax", + "language_model.model.layers.27.self_attn.attn.impl.matmul_av", + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.27.self_attn.attn.impl.k_cache", + "language_model.model.layers.27.self_attn.attn.impl.v_cache", + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.27.feed_forward.router", + "language_model.model.layers.27.feed_forward.experts", + "language_model.model.layers.27.feed_forward.experts.moe_op", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.27.feed_forward.shared_expert.down_proj", + "language_model.model.layers.28.self_attn.qkv_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.28.self_attn.attn.impl.softmax", + "language_model.model.layers.28.self_attn.attn.impl.matmul_av", + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.28.self_attn.attn.impl.k_cache", + "language_model.model.layers.28.self_attn.attn.impl.v_cache", + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.28.feed_forward.gate_up_proj", + "language_model.model.layers.28.feed_forward.down_proj", + "language_model.model.layers.29.self_attn.qkv_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.29.self_attn.attn.impl.softmax", + "language_model.model.layers.29.self_attn.attn.impl.matmul_av", + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.29.self_attn.attn.impl.k_cache", + "language_model.model.layers.29.self_attn.attn.impl.v_cache", + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.29.feed_forward.router", + "language_model.model.layers.29.feed_forward.experts", + "language_model.model.layers.29.feed_forward.experts.moe_op", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.29.feed_forward.shared_expert.down_proj", + "language_model.model.layers.30.self_attn.qkv_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.30.self_attn.attn.impl.softmax", + "language_model.model.layers.30.self_attn.attn.impl.matmul_av", + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.30.self_attn.attn.impl.k_cache", + "language_model.model.layers.30.self_attn.attn.impl.v_cache", + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.30.feed_forward.gate_up_proj", + "language_model.model.layers.30.feed_forward.down_proj", + "language_model.model.layers.31.self_attn.qkv_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.31.self_attn.attn.impl.softmax", + "language_model.model.layers.31.self_attn.attn.impl.matmul_av", + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.31.self_attn.attn.impl.k_cache", + "language_model.model.layers.31.self_attn.attn.impl.v_cache", + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.31.feed_forward.router", + "language_model.model.layers.31.feed_forward.experts", + "language_model.model.layers.31.feed_forward.experts.moe_op", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.31.feed_forward.shared_expert.down_proj", + "language_model.model.layers.32.self_attn.qkv_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.32.self_attn.attn.impl.softmax", + "language_model.model.layers.32.self_attn.attn.impl.matmul_av", + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.32.self_attn.attn.impl.k_cache", + "language_model.model.layers.32.self_attn.attn.impl.v_cache", + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.32.feed_forward.gate_up_proj", + "language_model.model.layers.32.feed_forward.down_proj", + "language_model.model.layers.33.self_attn.qkv_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.33.self_attn.attn.impl.softmax", + "language_model.model.layers.33.self_attn.attn.impl.matmul_av", + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.33.self_attn.attn.impl.k_cache", + "language_model.model.layers.33.self_attn.attn.impl.v_cache", + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.33.feed_forward.router", + "language_model.model.layers.33.feed_forward.experts", + "language_model.model.layers.33.feed_forward.experts.moe_op", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.33.feed_forward.shared_expert.down_proj", + "language_model.model.layers.34.self_attn.qkv_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.34.self_attn.attn.impl.softmax", + "language_model.model.layers.34.self_attn.attn.impl.matmul_av", + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.34.self_attn.attn.impl.k_cache", + "language_model.model.layers.34.self_attn.attn.impl.v_cache", + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.34.feed_forward.gate_up_proj", + "language_model.model.layers.34.feed_forward.down_proj", + "language_model.model.layers.35.self_attn.qkv_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.35.self_attn.attn.impl.softmax", + "language_model.model.layers.35.self_attn.attn.impl.matmul_av", + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.35.self_attn.attn.impl.k_cache", + "language_model.model.layers.35.self_attn.attn.impl.v_cache", + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.35.feed_forward.router", + "language_model.model.layers.35.feed_forward.experts", + "language_model.model.layers.35.feed_forward.experts.moe_op", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.35.feed_forward.shared_expert.down_proj", + "language_model.model.layers.36.self_attn.qkv_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.36.self_attn.attn.impl.softmax", + "language_model.model.layers.36.self_attn.attn.impl.matmul_av", + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.36.self_attn.attn.impl.k_cache", + "language_model.model.layers.36.self_attn.attn.impl.v_cache", + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.36.feed_forward.gate_up_proj", + "language_model.model.layers.36.feed_forward.down_proj", + "language_model.model.layers.37.self_attn.qkv_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.37.self_attn.attn.impl.softmax", + "language_model.model.layers.37.self_attn.attn.impl.matmul_av", + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.37.self_attn.attn.impl.k_cache", + "language_model.model.layers.37.self_attn.attn.impl.v_cache", + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.37.feed_forward.router", + "language_model.model.layers.37.feed_forward.experts", + "language_model.model.layers.37.feed_forward.experts.moe_op", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.37.feed_forward.shared_expert.down_proj", + "language_model.model.layers.38.self_attn.qkv_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.38.self_attn.attn.impl.softmax", + "language_model.model.layers.38.self_attn.attn.impl.matmul_av", + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.38.self_attn.attn.impl.k_cache", + "language_model.model.layers.38.self_attn.attn.impl.v_cache", + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.38.feed_forward.gate_up_proj", + "language_model.model.layers.38.feed_forward.down_proj", + "language_model.model.layers.39.self_attn.qkv_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.39.self_attn.attn.impl.softmax", + "language_model.model.layers.39.self_attn.attn.impl.matmul_av", + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.39.self_attn.attn.impl.k_cache", + "language_model.model.layers.39.self_attn.attn.impl.v_cache", + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.39.feed_forward.router", + "language_model.model.layers.39.feed_forward.experts", + "language_model.model.layers.39.feed_forward.experts.moe_op", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.39.feed_forward.shared_expert.down_proj", + "language_model.model.layers.40.self_attn.qkv_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.40.self_attn.attn.impl.softmax", + "language_model.model.layers.40.self_attn.attn.impl.matmul_av", + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.40.self_attn.attn.impl.k_cache", + "language_model.model.layers.40.self_attn.attn.impl.v_cache", + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.40.feed_forward.gate_up_proj", + "language_model.model.layers.40.feed_forward.down_proj", + "language_model.model.layers.41.self_attn.qkv_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.41.self_attn.attn.impl.softmax", + "language_model.model.layers.41.self_attn.attn.impl.matmul_av", + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.41.self_attn.attn.impl.k_cache", + "language_model.model.layers.41.self_attn.attn.impl.v_cache", + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.41.feed_forward.router", + "language_model.model.layers.41.feed_forward.experts", + "language_model.model.layers.41.feed_forward.experts.moe_op", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.41.feed_forward.shared_expert.down_proj", + "language_model.model.layers.42.self_attn.qkv_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.42.self_attn.attn.impl.softmax", + "language_model.model.layers.42.self_attn.attn.impl.matmul_av", + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.42.self_attn.attn.impl.k_cache", + "language_model.model.layers.42.self_attn.attn.impl.v_cache", + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.42.feed_forward.gate_up_proj", + "language_model.model.layers.42.feed_forward.down_proj", + "language_model.model.layers.43.self_attn.qkv_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.43.self_attn.attn.impl.softmax", + "language_model.model.layers.43.self_attn.attn.impl.matmul_av", + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.43.self_attn.attn.impl.k_cache", + "language_model.model.layers.43.self_attn.attn.impl.v_cache", + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.43.feed_forward.router", + "language_model.model.layers.43.feed_forward.experts", + "language_model.model.layers.43.feed_forward.experts.moe_op", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.43.feed_forward.shared_expert.down_proj", + "language_model.model.layers.44.self_attn.qkv_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.44.self_attn.attn.impl.softmax", + "language_model.model.layers.44.self_attn.attn.impl.matmul_av", + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.44.self_attn.attn.impl.k_cache", + "language_model.model.layers.44.self_attn.attn.impl.v_cache", + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.44.feed_forward.gate_up_proj", + "language_model.model.layers.44.feed_forward.down_proj", + "language_model.model.layers.45.self_attn.qkv_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.45.self_attn.attn.impl.softmax", + "language_model.model.layers.45.self_attn.attn.impl.matmul_av", + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.45.self_attn.attn.impl.k_cache", + "language_model.model.layers.45.self_attn.attn.impl.v_cache", + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.45.feed_forward.router", + "language_model.model.layers.45.feed_forward.experts", + "language_model.model.layers.45.feed_forward.experts.moe_op", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.45.feed_forward.shared_expert.down_proj", + "language_model.model.layers.46.self_attn.qkv_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.46.self_attn.attn.impl.softmax", + "language_model.model.layers.46.self_attn.attn.impl.matmul_av", + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.46.self_attn.attn.impl.k_cache", + "language_model.model.layers.46.self_attn.attn.impl.v_cache", + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.46.feed_forward.gate_up_proj", + "language_model.model.layers.46.feed_forward.down_proj", + "language_model.model.layers.47.self_attn.qkv_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.47.self_attn.attn.impl.softmax", + "language_model.model.layers.47.self_attn.attn.impl.matmul_av", + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.47.self_attn.attn.impl.k_cache", + "language_model.model.layers.47.self_attn.attn.impl.v_cache", + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.47.feed_forward.router", + "language_model.model.layers.47.feed_forward.experts", + "language_model.model.layers.47.feed_forward.experts.moe_op", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.47.feed_forward.shared_expert.down_proj", + "language_model.lm_head" +] \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_4_8.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_4_8.json new file mode 100644 index 000000000000..e241a52f71e5 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_4_8.json @@ -0,0 +1,24263 @@ +{ + "GlobalRank": null, + "LocalRank": 4, + "Mode": "DynamicRange", + "Nodes": { + "language_model.model.layers.0.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 11.1875 + ] + ] + ], + "params": { + "weight": [ + [ + 1.921875 + ] + ] + } + }, + "language_model.model.layers.0.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.76171875 + ] + ] + ], + "outputs": [ + [ + [ + 0.83203125 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 2.203125 + ] + ], + [ + [ + 10.5 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.8046875 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.203125 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.8046875 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 10.5 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.8046875 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 28.5 + ] + ], + [ + [ + 10.5 + ] + ], + [ + [ + 0.8046875 + ] + ] + ], + "outputs": [ + [ + [ + 0.76171875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.0.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 2.546875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.0.feed_forward.down_proj": { + "inputs": [ + [ + [ + 3.625 + ] + ] + ], + "outputs": [ + [ + [ + 26.75 + ] + ], + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.578125 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 10.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.287109375 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.765625 + ] + ] + ], + "outputs": [ + [ + [ + 0.67578125 + ] + ], + [ + [ + 1.3984375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.361328125 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.58984375 + ] + ], + [ + [ + 8.5625 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.796875 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.58984375 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.96875 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 8.5625 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.796875 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.0625 + ] + ], + [ + [ + 8.5625 + ] + ], + [ + [ + 1.796875 + ] + ] + ], + "outputs": [ + [ + [ + 0.74609375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.1.feed_forward.router": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "outputs": [ + [ + [ + 1.5390625 + ] + ], + [ + [ + 19.25 + ] + ], + [ + [ + 19.125 + ] + ], + [ + [ + 11.375 + ] + ], + [ + [ + 17.25 + ] + ], + [ + [ + 13.3125 + ] + ], + [ + [ + 21.125 + ] + ], + [ + [ + 14.5 + ] + ], + [ + [ + 22.375 + ] + ], + [ + [ + 19.125 + ] + ], + [ + [ + 19.75 + ] + ], + [ + [ + 15.0 + ] + ], + [ + [ + 13.75 + ] + ], + [ + [ + 14.875 + ] + ], + [ + [ + 18.25 + ] + ], + [ + [ + 10.625 + ] + ], + [ + [ + 14.0 + ] + ] + ] + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10888671875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.373046875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3046875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 6.3125 + ] + ] + ], + "outputs": [ + [ + [ + 0.4921875 + ] + ], + [ + [ + 4.286760286406354e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 9.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.83203125 + ] + ] + ], + "outputs": [ + [ + [ + 0.7734375 + ] + ], + [ + [ + 1.4921875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.60546875 + ] + ], + [ + [ + 11.0625 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.91796875 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.60546875 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.72265625 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 11.0625 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.91796875 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.875 + ] + ], + [ + [ + 11.0625 + ] + ], + [ + [ + 0.91796875 + ] + ] + ], + "outputs": [ + [ + [ + 0.83203125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.2.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 6.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.2.feed_forward.down_proj": { + "inputs": [ + [ + [ + 6.71875 + ] + ] + ], + "outputs": [ + [ + [ + 4.75 + ] + ], + [ + [ + 25.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5703125 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 7.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4609375 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.2421875 + ] + ] + ], + "outputs": [ + [ + [ + 0.77734375 + ] + ], + [ + [ + 2.40625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.0791015625 + ] + ], + [ + [ + 3.28125 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.33984375 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.0791015625 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.2080078125 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 3.28125 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.33984375 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 0.8671875 + ] + ], + [ + [ + 3.28125 + ] + ], + [ + [ + 0.33984375 + ] + ] + ], + "outputs": [ + [ + [ + 0.2421875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.3.feed_forward.router": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.52734375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "outputs": [ + [ + [ + 1.5703125 + ] + ], + [ + [ + 20.75 + ] + ], + [ + [ + 17.625 + ] + ], + [ + [ + 18.75 + ] + ], + [ + [ + 26.5 + ] + ], + [ + [ + 19.75 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 22.375 + ] + ], + [ + [ + 24.5 + ] + ], + [ + [ + 16.375 + ] + ], + [ + [ + 13.125 + ] + ], + [ + [ + 9.0 + ] + ], + [ + [ + 27.875 + ] + ], + [ + [ + 19.75 + ] + ], + [ + [ + 6.4375 + ] + ], + [ + [ + 16.375 + ] + ], + [ + [ + 27.375 + ] + ] + ] + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1005859375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11279296875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.103515625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10986328125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08349609375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08984375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11279296875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08349609375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.111328125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 200.0 + ] + ] + ], + "outputs": [ + [ + [ + 175.0 + ] + ], + [ + [ + 3.7218383881977644e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.875 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 14.3125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23828125 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.71875 + ] + ] + ], + "outputs": [ + [ + [ + 1.46875 + ] + ], + [ + [ + 2.328125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.72265625 + ] + ], + [ + [ + 7.875 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.25 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.72265625 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.71875 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 7.875 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.25 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.0 + ] + ], + [ + [ + 7.875 + ] + ], + [ + [ + 4.0625 + ] + ] + ], + "outputs": [ + [ + [ + 3.578125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.4.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 6.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.4.feed_forward.down_proj": { + "inputs": [ + [ + [ + 6.8125 + ] + ] + ], + "outputs": [ + [ + [ + 2.796875 + ] + ], + [ + [ + 5.1875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.462890625 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 18.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.8984375 + ] + ] + ], + "outputs": [ + [ + [ + 1.4921875 + ] + ], + [ + [ + 2.40625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.80078125 + ] + ], + [ + [ + 15.75 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.359375 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.80078125 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4765625 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.75 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.359375 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.25 + ] + ], + [ + [ + 15.75 + ] + ], + [ + [ + 2.359375 + ] + ] + ], + "outputs": [ + [ + [ + 1.8984375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.5.feed_forward.router": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 3.09375 + ] + ], + [ + [ + 12.6875 + ] + ], + [ + [ + 36.25 + ] + ], + [ + [ + 13.0625 + ] + ], + [ + [ + 28.25 + ] + ], + [ + [ + 42.25 + ] + ], + [ + [ + 16.375 + ] + ], + [ + [ + 9.9375 + ] + ], + [ + [ + 21.875 + ] + ], + [ + [ + 35.75 + ] + ], + [ + [ + 24.875 + ] + ], + [ + [ + 11.4375 + ] + ], + [ + [ + 31.75 + ] + ], + [ + [ + 25.375 + ] + ], + [ + [ + 21.25 + ] + ], + [ + [ + 36.25 + ] + ], + [ + [ + 28.375 + ] + ] + ] + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.111328125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10546875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.69921875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1025390625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10986328125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10546875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 5.28125 + ] + ] + ], + "outputs": [ + [ + [ + 0.9609375 + ] + ], + [ + [ + 3.705223038250453e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.369140625 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 14.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.75 + ] + ] + ], + "outputs": [ + [ + [ + 0.94140625 + ] + ], + [ + [ + 1.96875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.73828125 + ] + ], + [ + [ + 15.6875 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.09375 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.73828125 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.71875 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.6875 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.09375 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.0625 + ] + ], + [ + [ + 15.6875 + ] + ], + [ + [ + 4.09375 + ] + ] + ], + "outputs": [ + [ + [ + 2.515625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.6.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 7.71875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.6.feed_forward.down_proj": { + "inputs": [ + [ + [ + 7.8125 + ] + ] + ], + "outputs": [ + [ + [ + 2.25 + ] + ], + [ + [ + 11.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37109375 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 7.59375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.82421875 + ] + ] + ], + "outputs": [ + [ + [ + 2.1875 + ] + ], + [ + [ + 2.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.455078125 + ] + ], + [ + [ + 6.3125 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.390625 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.455078125 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.64453125 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 6.3125 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.390625 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 5.21875 + ] + ], + [ + [ + 6.3125 + ] + ], + [ + [ + 1.3359375 + ] + ] + ], + "outputs": [ + [ + [ + 0.82421875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.7.feed_forward.router": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "outputs": [ + [ + [ + 2.96875 + ] + ], + [ + [ + 45.5 + ] + ], + [ + [ + 17.875 + ] + ], + [ + [ + 29.375 + ] + ], + [ + [ + 19.375 + ] + ], + [ + [ + 14.0625 + ] + ], + [ + [ + 52.0 + ] + ], + [ + [ + 39.25 + ] + ], + [ + [ + 28.625 + ] + ], + [ + [ + 21.875 + ] + ], + [ + [ + 30.375 + ] + ], + [ + [ + 36.0 + ] + ], + [ + [ + 7.875 + ] + ], + [ + [ + 17.875 + ] + ], + [ + [ + 31.25 + ] + ], + [ + [ + 20.625 + ] + ], + [ + [ + 30.375 + ] + ] + ] + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10546875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.39453125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08642578125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.091796875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.62109375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.61328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 11.5625 + ] + ] + ], + "outputs": [ + [ + [ + 3.5 + ] + ], + [ + [ + 4.286760286406354e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3046875 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.140625 + ] + ] + ], + "outputs": [ + [ + [ + 2.265625 + ] + ], + [ + [ + 3.390625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.796875 + ] + ], + [ + [ + 10.75 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.359375 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.796875 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.90625 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 10.75 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.359375 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.25 + ] + ], + [ + [ + 10.75 + ] + ], + [ + [ + 3.359375 + ] + ] + ], + "outputs": [ + [ + [ + 3.140625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.8.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 8.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.287109375 + ] + ] + } + }, + "language_model.model.layers.8.feed_forward.down_proj": { + "inputs": [ + [ + [ + 10.25 + ] + ] + ], + "outputs": [ + [ + [ + 2.84375 + ] + ], + [ + [ + 9.8125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.55078125 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 16.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.84375 + ] + ] + ], + "outputs": [ + [ + [ + 1.609375 + ] + ], + [ + [ + 3.640625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.91796875 + ] + ], + [ + [ + 17.875 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.859375 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.91796875 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.84375 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.875 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.859375 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.125 + ] + ], + [ + [ + 17.875 + ] + ], + [ + [ + 2.859375 + ] + ] + ], + "outputs": [ + [ + [ + 2.84375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.9.feed_forward.router": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "outputs": [ + [ + [ + 5.0 + ] + ], + [ + [ + 23.25 + ] + ], + [ + [ + 26.75 + ] + ], + [ + [ + 64.5 + ] + ], + [ + [ + 62.75 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 28.625 + ] + ], + [ + [ + 35.25 + ] + ], + [ + [ + 38.75 + ] + ], + [ + [ + 24.375 + ] + ], + [ + [ + 20.875 + ] + ], + [ + [ + 45.25 + ] + ], + [ + [ + 29.5 + ] + ], + [ + [ + 69.0 + ] + ], + [ + [ + 40.25 + ] + ], + [ + [ + 46.5 + ] + ], + [ + [ + 60.0 + ] + ] + ] + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11474609375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11181640625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0869140625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1005859375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.083984375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3046875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 11.5625 + ] + ] + ], + "outputs": [ + [ + [ + 3.0625 + ] + ], + [ + [ + 4.286760286406354e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 17.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.671875 + ] + ] + ], + "outputs": [ + [ + [ + 1.984375 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0390625 + ] + ], + [ + [ + 18.75 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.125 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0390625 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.609375 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.75 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.125 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.625 + ] + ], + [ + [ + 18.75 + ] + ], + [ + [ + 3.125 + ] + ] + ], + "outputs": [ + [ + [ + 2.671875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.10.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 9.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.10.feed_forward.down_proj": { + "inputs": [ + [ + [ + 18.875 + ] + ] + ], + "outputs": [ + [ + [ + 3.40625 + ] + ], + [ + [ + 12.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 15.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11279296875 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.921875 + ] + ] + ], + "outputs": [ + [ + [ + 8.625 + ] + ], + [ + [ + 8.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.921875 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.8046875 + ] + ], + [ + [ + 8.125 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.5625 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.8046875 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.8125 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 8.125 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.5625 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.125 + ] + ], + [ + [ + 8.125 + ] + ], + [ + [ + 5.53125 + ] + ] + ], + "outputs": [ + [ + [ + 3.921875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.11.feed_forward.router": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "outputs": [ + [ + [ + 4.5625 + ] + ], + [ + [ + 46.0 + ] + ], + [ + [ + 27.25 + ] + ], + [ + [ + 65.5 + ] + ], + [ + [ + 72.5 + ] + ], + [ + [ + 45.5 + ] + ], + [ + [ + 59.0 + ] + ], + [ + [ + 33.0 + ] + ], + [ + [ + 61.0 + ] + ], + [ + [ + 28.5 + ] + ], + [ + [ + 64.5 + ] + ], + [ + [ + 49.5 + ] + ], + [ + [ + 57.25 + ] + ], + [ + [ + 58.5 + ] + ], + [ + [ + 25.375 + ] + ], + [ + [ + 64.0 + ] + ], + [ + [ + 41.5 + ] + ] + ] + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10986328125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0859375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09130859375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10498046875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11376953125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3828125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 15.0 + ] + ] + ], + "outputs": [ + [ + [ + 1.390625 + ] + ], + [ + [ + 3.705223038250453e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.359375 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 20.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.75 + ] + ] + ], + "outputs": [ + [ + [ + 1.796875 + ] + ], + [ + [ + 3.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.87109375 + ] + ], + [ + [ + 13.6875 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.78125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.87109375 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.515625 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.6875 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.78125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.6875 + ] + ], + [ + [ + 13.6875 + ] + ], + [ + [ + 3.78125 + ] + ] + ], + "outputs": [ + [ + [ + 2.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.12.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 10.6875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.12.feed_forward.down_proj": { + "inputs": [ + [ + [ + 20.25 + ] + ] + ], + "outputs": [ + [ + [ + 9.25 + ] + ], + [ + [ + 20.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.67578125 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.03125 + ] + ] + ], + "outputs": [ + [ + [ + 2.015625 + ] + ], + [ + [ + 6.34375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.82421875 + ] + ], + [ + [ + 16.875 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.5625 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.82421875 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.28125 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.875 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.5625 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.5 + ] + ], + [ + [ + 16.5 + ] + ], + [ + [ + 4.5625 + ] + ] + ], + "outputs": [ + [ + [ + 4.03125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.13.feed_forward.router": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "outputs": [ + [ + [ + 15.8125 + ] + ], + [ + [ + 51.5 + ] + ], + [ + [ + 61.25 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 41.25 + ] + ], + [ + [ + 47.0 + ] + ], + [ + [ + 19.125 + ] + ], + [ + [ + 92.5 + ] + ], + [ + [ + 65.5 + ] + ], + [ + [ + 75.5 + ] + ], + [ + [ + 36.75 + ] + ], + [ + [ + 42.25 + ] + ], + [ + [ + 41.25 + ] + ], + [ + [ + 62.5 + ] + ], + [ + [ + 51.5 + ] + ], + [ + [ + 54.75 + ] + ], + [ + [ + 37.0 + ] + ] + ] + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.384765625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5234375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 15.6875 + ] + ] + ], + "outputs": [ + [ + [ + 6.03125 + ] + ], + [ + [ + 4.286760286406354e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.376953125 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 20.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.0 + ] + ] + ], + "outputs": [ + [ + [ + 3.546875 + ] + ], + [ + [ + 5.71875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0078125 + ] + ], + [ + [ + 18.375 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.0 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0078125 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.15625 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.375 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.0 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.0625 + ] + ], + [ + [ + 18.125 + ] + ], + [ + [ + 6.0 + ] + ] + ], + "outputs": [ + [ + [ + 6.0 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.14.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 12.0625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.14.feed_forward.down_proj": { + "inputs": [ + [ + [ + 13.0 + ] + ] + ], + "outputs": [ + [ + [ + 5.375 + ] + ], + [ + [ + 20.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 23.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.46875 + ] + ] + ], + "outputs": [ + [ + [ + 20.0 + ] + ], + [ + [ + 21.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.8046875 + ] + ], + [ + [ + 11.25 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.46875 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.8046875 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.40625 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 11.25 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.46875 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.125 + ] + ], + [ + [ + 11.25 + ] + ], + [ + [ + 7.46875 + ] + ] + ], + "outputs": [ + [ + [ + 6.46875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.15.feed_forward.router": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 5.84375 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 61.75 + ] + ], + [ + [ + 75.5 + ] + ], + [ + [ + 78.0 + ] + ], + [ + [ + 42.25 + ] + ], + [ + [ + 62.5 + ] + ], + [ + [ + 56.25 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 83.0 + ] + ], + [ + [ + 83.0 + ] + ], + [ + [ + 50.75 + ] + ], + [ + [ + 44.75 + ] + ], + [ + [ + 67.5 + ] + ], + [ + [ + 70.5 + ] + ], + [ + [ + 88.5 + ] + ], + [ + [ + 57.25 + ] + ] + ] + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11767578125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2353515625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1005859375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08642578125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.103515625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 23.0 + ] + ] + ], + "outputs": [ + [ + [ + 1.4140625 + ] + ], + [ + [ + 4.286760286406354e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 23.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.0625 + ] + ] + ], + "outputs": [ + [ + [ + 2.4375 + ] + ], + [ + [ + 6.3125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.8359375 + ] + ], + [ + [ + 13.0625 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.78125 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.8359375 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.015625 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.0625 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.78125 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.25 + ] + ], + [ + [ + 13.0625 + ] + ], + [ + [ + 4.78125 + ] + ] + ], + "outputs": [ + [ + [ + 4.0625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.16.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 14.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.16.feed_forward.down_proj": { + "inputs": [ + [ + [ + 18.0 + ] + ] + ], + "outputs": [ + [ + [ + 6.65625 + ] + ], + [ + [ + 27.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.57421875 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.765625 + ] + ] + ], + "outputs": [ + [ + [ + 19.5 + ] + ], + [ + [ + 27.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.91015625 + ] + ], + [ + [ + 13.5 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.40625 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.91015625 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.296875 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.5 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.40625 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.125 + ] + ], + [ + [ + 13.5 + ] + ], + [ + [ + 4.40625 + ] + ] + ], + "outputs": [ + [ + [ + 3.765625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.17.feed_forward.router": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 6.8125 + ] + ], + [ + [ + 74.0 + ] + ], + [ + [ + 91.5 + ] + ], + [ + [ + 31.125 + ] + ], + [ + [ + 60.25 + ] + ], + [ + [ + 85.5 + ] + ], + [ + [ + 53.75 + ] + ], + [ + [ + 54.5 + ] + ], + [ + [ + 77.5 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 82.5 + ] + ], + [ + [ + 38.5 + ] + ], + [ + [ + 78.0 + ] + ], + [ + [ + 82.5 + ] + ], + [ + [ + 76.0 + ] + ], + [ + [ + 43.75 + ] + ], + [ + [ + 53.25 + ] + ] + ] + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23828125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.244140625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11572265625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.091796875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2490234375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.103515625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 28.75 + ] + ] + ], + "outputs": [ + [ + [ + 7.9375 + ] + ], + [ + [ + 4.286760286406354e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.53125 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.59375 + ] + ] + ], + "outputs": [ + [ + [ + 6.625 + ] + ], + [ + [ + 12.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4765625 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1015625 + ] + ], + [ + [ + 18.0 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.90625 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1015625 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.25 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.0 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.90625 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.25 + ] + ], + [ + [ + 18.0 + ] + ], + [ + [ + 7.78125 + ] + ] + ], + "outputs": [ + [ + [ + 7.59375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.18.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 20.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.18.feed_forward.down_proj": { + "inputs": [ + [ + [ + 19.25 + ] + ] + ], + "outputs": [ + [ + [ + 8.1875 + ] + ], + [ + [ + 354.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6875 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 44.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11279296875 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.59375 + ] + ] + ], + "outputs": [ + [ + [ + 31.5 + ] + ], + [ + [ + 37.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6796875 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.8671875 + ] + ], + [ + [ + 12.875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.8671875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.21875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 12.875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.0 + ] + ], + [ + [ + 12.875 + ] + ], + [ + [ + 5.875 + ] + ] + ], + "outputs": [ + [ + [ + 4.59375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.19.feed_forward.router": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "outputs": [ + [ + [ + 7.25 + ] + ], + [ + [ + 61.25 + ] + ], + [ + [ + 73.0 + ] + ], + [ + [ + 76.5 + ] + ], + [ + [ + 40.75 + ] + ], + [ + [ + 94.5 + ] + ], + [ + [ + 42.75 + ] + ], + [ + [ + 66.5 + ] + ], + [ + [ + 90.0 + ] + ], + [ + [ + 120.0 + ] + ], + [ + [ + 46.25 + ] + ], + [ + [ + 48.25 + ] + ], + [ + [ + 37.5 + ] + ], + [ + [ + 31.5 + ] + ], + [ + [ + 42.0 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 66.5 + ] + ] + ] + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.44921875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2490234375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.349609375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.404296875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.89453125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.359375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.419921875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.35546875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 29.125 + ] + ] + ], + "outputs": [ + [ + [ + 6.34375 + ] + ], + [ + [ + 3.705223038250453e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.455078125 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 53.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.875 + ] + ] + ], + "outputs": [ + [ + [ + 12.8125 + ] + ], + [ + [ + 22.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.984375 + ] + ], + [ + [ + 19.375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.84375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.984375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.5 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.84375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.3125 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 7.84375 + ] + ] + ], + "outputs": [ + [ + [ + 4.875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.20.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 42.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.38671875 + ] + ] + } + }, + "language_model.model.layers.20.feed_forward.down_proj": { + "inputs": [ + [ + [ + 15.4375 + ] + ] + ], + "outputs": [ + [ + [ + 10.5 + ] + ], + [ + [ + 41.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.77734375 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 58.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.84375 + ] + ] + ], + "outputs": [ + [ + [ + 26.0 + ] + ], + [ + [ + 35.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.99609375 + ] + ], + [ + [ + 20.75 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.78125 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.99609375 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.21875 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.75 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.78125 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.8125 + ] + ], + [ + [ + 20.75 + ] + ], + [ + [ + 5.78125 + ] + ] + ], + "outputs": [ + [ + [ + 4.84375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.21.feed_forward.router": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.45703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "outputs": [ + [ + [ + 8.625 + ] + ], + [ + [ + 34.5 + ] + ], + [ + [ + 74.0 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 24.25 + ] + ], + [ + [ + 56.25 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 59.75 + ] + ], + [ + [ + 68.0 + ] + ], + [ + [ + 94.0 + ] + ], + [ + [ + 47.5 + ] + ], + [ + [ + 63.5 + ] + ], + [ + [ + 86.5 + ] + ], + [ + [ + 64.5 + ] + ], + [ + [ + 37.0 + ] + ], + [ + [ + 80.0 + ] + ], + [ + [ + 62.5 + ] + ] + ] + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 25.625 + ] + ] + ], + "outputs": [ + [ + [ + 21.75 + ] + ], + [ + [ + 3.705223038250453e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.890625 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 57.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.46875 + ] + ] + ], + "outputs": [ + [ + [ + 15.75 + ] + ], + [ + [ + 28.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1328125 + ] + ], + [ + [ + 20.375 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.3125 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1328125 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.6875 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.375 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.3125 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.25 + ] + ], + [ + [ + 20.125 + ] + ], + [ + [ + 7.3125 + ] + ] + ], + "outputs": [ + [ + [ + 4.46875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.22.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 46.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.22.feed_forward.down_proj": { + "inputs": [ + [ + [ + 25.125 + ] + ] + ], + "outputs": [ + [ + [ + 14.875 + ] + ], + [ + [ + 77.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.734375 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 69.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.65625 + ] + ] + ], + "outputs": [ + [ + [ + 70.0 + ] + ], + [ + [ + 79.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.388671875 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.140625 + ] + ], + [ + [ + 15.5625 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.375 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.140625 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.1875 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.5625 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.375 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.25 + ] + ], + [ + [ + 15.5625 + ] + ], + [ + [ + 8.375 + ] + ] + ], + "outputs": [ + [ + [ + 7.65625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.23.feed_forward.router": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.474609375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 8.75 + ] + ], + [ + [ + 18.25 + ] + ], + [ + [ + 54.75 + ] + ], + [ + [ + 67.5 + ] + ], + [ + [ + 104.0 + ] + ], + [ + [ + 60.75 + ] + ], + [ + [ + 56.0 + ] + ], + [ + [ + 77.5 + ] + ], + [ + [ + 63.0 + ] + ], + [ + [ + 65.5 + ] + ], + [ + [ + 49.0 + ] + ], + [ + [ + 48.25 + ] + ], + [ + [ + 55.5 + ] + ], + [ + [ + 114.5 + ] + ], + [ + [ + 59.5 + ] + ], + [ + [ + 73.0 + ] + ], + [ + [ + 32.0 + ] + ] + ] + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.474609375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.703125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 27.0 + ] + ] + ], + "outputs": [ + [ + [ + 10.625 + ] + ], + [ + [ + 4.286760286406354e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.470703125 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.03125 + ] + ] + ], + "outputs": [ + [ + [ + 48.0 + ] + ], + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.359375 + ] + ], + [ + [ + 17.25 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.0 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.359375 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.0 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.25 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.0 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 17.25 + ] + ], + [ + [ + 17.25 + ] + ], + [ + [ + 8.0 + ] + ] + ], + "outputs": [ + [ + [ + 6.03125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.24.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.380859375 + ] + ] + } + }, + "language_model.model.layers.24.feed_forward.down_proj": { + "inputs": [ + [ + [ + 21.25 + ] + ] + ], + "outputs": [ + [ + [ + 21.75 + ] + ], + [ + [ + 100.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.72265625 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.25 + ] + ] + ], + "outputs": [ + [ + [ + 53.0 + ] + ], + [ + [ + 69.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.47265625 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.5390625 + ] + ], + [ + [ + 16.0 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.6875 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.5390625 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.03125 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.0 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.6875 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 18.375 + ] + ], + [ + [ + 16.0 + ] + ], + [ + [ + 6.6875 + ] + ] + ], + "outputs": [ + [ + [ + 5.25 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.25.feed_forward.router": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.412109375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "outputs": [ + [ + [ + 12.875 + ] + ], + [ + [ + 114.5 + ] + ], + [ + [ + 23.25 + ] + ], + [ + [ + 46.0 + ] + ], + [ + [ + 67.5 + ] + ], + [ + [ + 44.5 + ] + ], + [ + [ + 45.25 + ] + ], + [ + [ + 36.75 + ] + ], + [ + [ + 89.0 + ] + ], + [ + [ + 125.5 + ] + ], + [ + [ + 59.75 + ] + ], + [ + [ + 71.5 + ] + ], + [ + [ + 82.5 + ] + ], + [ + [ + 39.5 + ] + ], + [ + [ + 87.5 + ] + ], + [ + [ + 68.0 + ] + ], + [ + [ + 57.5 + ] + ] + ] + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4296875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.419921875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 46.5 + ] + ] + ], + "outputs": [ + [ + [ + 40.5 + ] + ], + [ + [ + 3.705223038250453e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.80859375 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 63.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.75 + ] + ] + ], + "outputs": [ + [ + [ + 30.875 + ] + ], + [ + [ + 48.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.296875 + ] + ], + [ + [ + 20.625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.296875 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.6875 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.4375 + ] + ], + [ + [ + 20.625 + ] + ], + [ + [ + 9.625 + ] + ] + ], + "outputs": [ + [ + [ + 6.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.26.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 45.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.359375 + ] + ] + } + }, + "language_model.model.layers.26.feed_forward.down_proj": { + "inputs": [ + [ + [ + 28.625 + ] + ] + ], + "outputs": [ + [ + [ + 22.125 + ] + ], + [ + [ + 113.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.66796875 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.40625 + ] + ] + ], + "outputs": [ + [ + [ + 60.25 + ] + ], + [ + [ + 71.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1640625 + ] + ], + [ + [ + 15.9375 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.3125 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1640625 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.78125 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.9375 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.3125 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.875 + ] + ], + [ + [ + 15.6875 + ] + ], + [ + [ + 13.3125 + ] + ] + ], + "outputs": [ + [ + [ + 7.40625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.27.feed_forward.router": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "outputs": [ + [ + [ + 12.3125 + ] + ], + [ + [ + 65.5 + ] + ], + [ + [ + 132.0 + ] + ], + [ + [ + 48.25 + ] + ], + [ + [ + 66.5 + ] + ], + [ + [ + 29.25 + ] + ], + [ + [ + 111.0 + ] + ], + [ + [ + 62.75 + ] + ], + [ + [ + 77.5 + ] + ], + [ + [ + 139.0 + ] + ], + [ + [ + 73.0 + ] + ], + [ + [ + 49.5 + ] + ], + [ + [ + 96.0 + ] + ], + [ + [ + 41.75 + ] + ], + [ + [ + 82.0 + ] + ], + [ + [ + 55.0 + ] + ], + [ + [ + 84.0 + ] + ] + ] + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11474609375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 37.0 + ] + ] + ], + "outputs": [ + [ + [ + 13.875 + ] + ], + [ + [ + 4.286760286406354e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.83984375 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 78.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.53125 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 97.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.765625 + ] + ], + [ + [ + 17.875 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.53125 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.765625 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.5625 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.875 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.53125 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 20.375 + ] + ], + [ + [ + 17.875 + ] + ], + [ + [ + 7.53125 + ] + ] + ], + "outputs": [ + [ + [ + 6.53125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.28.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.431640625 + ] + ] + } + }, + "language_model.model.layers.28.feed_forward.down_proj": { + "inputs": [ + [ + [ + 28.625 + ] + ] + ], + "outputs": [ + [ + [ + 140.0 + ] + ], + [ + [ + 149.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.36328125 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 67.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.o_proj": { + "inputs": [ + [ + [ + 9.0625 + ] + ] + ], + "outputs": [ + [ + [ + 78.0 + ] + ], + [ + [ + 84.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.140625 + ] + ], + [ + [ + 17.25 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.0625 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.140625 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.9375 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.25 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.0625 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.0625 + ] + ], + [ + [ + 17.25 + ] + ], + [ + [ + 13.0625 + ] + ] + ], + "outputs": [ + [ + [ + 9.0625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.29.feed_forward.router": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.333984375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "outputs": [ + [ + [ + 11.4375 + ] + ], + [ + [ + 92.0 + ] + ], + [ + [ + 62.0 + ] + ], + [ + [ + 62.0 + ] + ], + [ + [ + 64.0 + ] + ], + [ + [ + 76.0 + ] + ], + [ + [ + 64.5 + ] + ], + [ + [ + 54.5 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 80.5 + ] + ], + [ + [ + 78.0 + ] + ], + [ + [ + 90.5 + ] + ], + [ + [ + 85.0 + ] + ], + [ + [ + 42.0 + ] + ], + [ + [ + 74.5 + ] + ], + [ + [ + 77.5 + ] + ], + [ + [ + 64.5 + ] + ] + ] + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.111328125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6171875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10693359375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.244140625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.41015625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 55.25 + ] + ] + ], + "outputs": [ + [ + [ + 17.625 + ] + ], + [ + [ + 4.286760286406354e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.87109375 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.1875 + ] + ] + ], + "outputs": [ + [ + [ + 50.25 + ] + ], + [ + [ + 56.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10546875 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.109375 + ] + ], + [ + [ + 22.5 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.5 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.109375 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.375 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 22.5 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.5 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.25 + ] + ], + [ + [ + 22.5 + ] + ], + [ + [ + 8.5 + ] + ] + ], + "outputs": [ + [ + [ + 7.1875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.30.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 32.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.392578125 + ] + ] + } + }, + "language_model.model.layers.30.feed_forward.down_proj": { + "inputs": [ + [ + [ + 112.5 + ] + ] + ], + "outputs": [ + [ + [ + 30.25 + ] + ], + [ + [ + 159.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.640625 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.75 + ] + ] + ], + "outputs": [ + [ + [ + 77.0 + ] + ], + [ + [ + 115.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 16.125 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.8125 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.34375 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.125 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 12.8125 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.1875 + ] + ], + [ + [ + 16.125 + ] + ], + [ + [ + 12.75 + ] + ] + ], + "outputs": [ + [ + [ + 6.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.31.feed_forward.router": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "outputs": [ + [ + [ + 40.25 + ] + ], + [ + [ + 116.5 + ] + ], + [ + [ + 35.0 + ] + ], + [ + [ + 55.0 + ] + ], + [ + [ + 63.0 + ] + ], + [ + [ + 69.5 + ] + ], + [ + [ + 36.0 + ] + ], + [ + [ + 150.0 + ] + ], + [ + [ + 58.5 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 79.5 + ] + ], + [ + [ + 30.125 + ] + ], + [ + [ + 42.0 + ] + ], + [ + [ + 70.5 + ] + ], + [ + [ + 136.0 + ] + ], + [ + [ + 73.0 + ] + ], + [ + [ + 81.5 + ] + ] + ] + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11376953125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10546875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4140625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.287109375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.546875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.455078125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "outputs": [ + [ + [ + 9.5 + ] + ], + [ + [ + 3.705223038250453e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.55859375 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 53.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.53125 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 109.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.369140625 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.625 + ] + ], + [ + [ + 19.0 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.09375 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.625 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.53125 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.0 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.09375 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 19.5 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 7.03125 + ] + ] + ], + "outputs": [ + [ + [ + 6.28125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.32.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 32.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.32.feed_forward.down_proj": { + "inputs": [ + [ + [ + 46.5 + ] + ] + ], + "outputs": [ + [ + [ + 29.375 + ] + ], + [ + [ + 75.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5625 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 44.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.25 + ] + ] + ], + "outputs": [ + [ + [ + 94.0 + ] + ], + [ + [ + 132.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.796875 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.6328125 + ] + ], + [ + [ + 19.125 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.875 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.6328125 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.9375 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.125 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.875 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 18.875 + ] + ], + [ + [ + 19.125 + ] + ], + [ + [ + 8.875 + ] + ] + ], + "outputs": [ + [ + [ + 8.25 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.33.feed_forward.router": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "outputs": [ + [ + [ + 19.625 + ] + ], + [ + [ + 71.5 + ] + ], + [ + [ + 103.5 + ] + ], + [ + [ + 94.5 + ] + ], + [ + [ + 23.75 + ] + ], + [ + [ + 86.5 + ] + ], + [ + [ + 54.25 + ] + ], + [ + [ + 67.0 + ] + ], + [ + [ + 74.5 + ] + ], + [ + [ + 106.5 + ] + ], + [ + [ + 86.0 + ] + ], + [ + [ + 80.0 + ] + ], + [ + [ + 112.0 + ] + ], + [ + [ + 105.5 + ] + ], + [ + [ + 122.0 + ] + ], + [ + [ + 71.0 + ] + ], + [ + [ + 139.0 + ] + ] + ] + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.455078125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.41796875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.330078125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 67.5 + ] + ] + ], + "outputs": [ + [ + [ + 13.75 + ] + ], + [ + [ + 3.705223038250453e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.470703125 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 54.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.46875 + ] + ] + ], + "outputs": [ + [ + [ + 55.75 + ] + ], + [ + [ + 98.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1025390625 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1640625 + ] + ], + [ + [ + 19.875 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.3125 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1640625 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.4375 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.875 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.3125 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.375 + ] + ], + [ + [ + 19.5 + ] + ], + [ + [ + 9.3125 + ] + ] + ], + "outputs": [ + [ + [ + 7.46875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.34.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.490234375 + ] + ] + } + }, + "language_model.model.layers.34.feed_forward.down_proj": { + "inputs": [ + [ + [ + 85.5 + ] + ] + ], + "outputs": [ + [ + [ + 36.5 + ] + ], + [ + [ + 162.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.80859375 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 54.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.25 + ] + ] + ], + "outputs": [ + [ + [ + 110.5 + ] + ], + [ + [ + 116.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.87109375 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.109375 + ] + ], + [ + [ + 16.25 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.3125 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.109375 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.5 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.25 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.3125 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.75 + ] + ], + [ + [ + 16.25 + ] + ], + [ + [ + 13.3125 + ] + ] + ], + "outputs": [ + [ + [ + 7.25 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.35.feed_forward.router": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "outputs": [ + [ + [ + 21.0 + ] + ], + [ + [ + 90.0 + ] + ], + [ + [ + 97.0 + ] + ], + [ + [ + 133.0 + ] + ], + [ + [ + 161.0 + ] + ], + [ + [ + 93.5 + ] + ], + [ + [ + 135.0 + ] + ], + [ + [ + 118.0 + ] + ], + [ + [ + 159.0 + ] + ], + [ + [ + 57.75 + ] + ], + [ + [ + 108.5 + ] + ], + [ + [ + 112.0 + ] + ], + [ + [ + 240.0 + ] + ], + [ + [ + 63.5 + ] + ], + [ + [ + 104.0 + ] + ], + [ + [ + 140.0 + ] + ], + [ + [ + 131.0 + ] + ] + ] + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11376953125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.341796875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.32421875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 80.5 + ] + ] + ], + "outputs": [ + [ + [ + 44.75 + ] + ], + [ + [ + 4.286760286406354e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.875 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 56.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.o_proj": { + "inputs": [ + [ + [ + 13.125 + ] + ] + ], + "outputs": [ + [ + [ + 36.75 + ] + ], + [ + [ + 57.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28515625 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.96875 + ] + ], + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.9375 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.96875 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.0 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 14.9375 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 23.375 + ] + ], + [ + [ + 18.625 + ] + ], + [ + [ + 14.9375 + ] + ] + ], + "outputs": [ + [ + [ + 13.125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.36.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 29.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.36.feed_forward.down_proj": { + "inputs": [ + [ + [ + 95.5 + ] + ] + ], + "outputs": [ + [ + [ + 43.0 + ] + ], + [ + [ + 94.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6953125 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.o_proj": { + "inputs": [ + [ + [ + 9.125 + ] + ] + ], + "outputs": [ + [ + [ + 97.5 + ] + ], + [ + [ + 150.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.390625 + ] + ], + [ + [ + 18.75 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.625 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.390625 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.125 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.75 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.625 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.875 + ] + ], + [ + [ + 18.5 + ] + ], + [ + [ + 10.625 + ] + ] + ], + "outputs": [ + [ + [ + 9.0 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.37.feed_forward.router": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "outputs": [ + [ + [ + 24.25 + ] + ], + [ + [ + 98.0 + ] + ], + [ + [ + 83.5 + ] + ], + [ + [ + 89.5 + ] + ], + [ + [ + 298.0 + ] + ], + [ + [ + 134.0 + ] + ], + [ + [ + 140.0 + ] + ], + [ + [ + 113.5 + ] + ], + [ + [ + 114.5 + ] + ], + [ + [ + 91.5 + ] + ], + [ + [ + 90.0 + ] + ], + [ + [ + 145.0 + ] + ], + [ + [ + 70.0 + ] + ], + [ + [ + 169.0 + ] + ], + [ + [ + 85.5 + ] + ], + [ + [ + 113.5 + ] + ], + [ + [ + 123.0 + ] + ] + ] + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.431640625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.240234375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.345703125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.35546875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 108.0 + ] + ] + ], + "outputs": [ + [ + [ + 18.0 + ] + ], + [ + [ + 4.286760286406354e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.82421875 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 47.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.283203125 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.0625 + ] + ] + ], + "outputs": [ + [ + [ + 29.75 + ] + ], + [ + [ + 66.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3984375 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.8125 + ] + ], + [ + [ + 13.6875 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.0 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.8125 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.375 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.6875 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.0 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 21.125 + ] + ], + [ + [ + 13.6875 + ] + ], + [ + [ + 9.0 + ] + ] + ], + "outputs": [ + [ + [ + 8.0625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.38.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 31.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.447265625 + ] + ] + } + }, + "language_model.model.layers.38.feed_forward.down_proj": { + "inputs": [ + [ + [ + 94.0 + ] + ] + ], + "outputs": [ + [ + [ + 31.125 + ] + ], + [ + [ + 262.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.439453125 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 52.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.875 + ] + ] + ], + "outputs": [ + [ + [ + 126.5 + ] + ], + [ + [ + 154.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.345703125 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.9609375 + ] + ], + [ + [ + 16.125 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.1875 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.9609375 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.25 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.125 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.1875 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.9375 + ] + ], + [ + [ + 15.875 + ] + ], + [ + [ + 9.1875 + ] + ] + ], + "outputs": [ + [ + [ + 6.875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.39.feed_forward.router": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "outputs": [ + [ + [ + 40.0 + ] + ], + [ + [ + 101.0 + ] + ], + [ + [ + 108.5 + ] + ], + [ + [ + 114.5 + ] + ], + [ + [ + 140.0 + ] + ], + [ + [ + 122.5 + ] + ], + [ + [ + 152.0 + ] + ], + [ + [ + 143.0 + ] + ], + [ + [ + 109.0 + ] + ], + [ + [ + 126.5 + ] + ], + [ + [ + 158.0 + ] + ], + [ + [ + 144.0 + ] + ], + [ + [ + 95.5 + ] + ], + [ + [ + 141.0 + ] + ], + [ + [ + 112.5 + ] + ], + [ + [ + 136.0 + ] + ], + [ + [ + 64.5 + ] + ] + ] + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.359375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09130859375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.578125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.244140625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 120.5 + ] + ] + ], + "outputs": [ + [ + [ + 49.0 + ] + ], + [ + [ + 4.286760286406354e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7734375 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 46.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.o_proj": { + "inputs": [ + [ + [ + 13.1875 + ] + ] + ], + "outputs": [ + [ + [ + 94.0 + ] + ], + [ + [ + 102.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.3125 + ] + ], + [ + [ + 16.0 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.8125 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.3125 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.375 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.0 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 14.8125 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.5625 + ] + ], + [ + [ + 16.0 + ] + ], + [ + [ + 14.8125 + ] + ] + ], + "outputs": [ + [ + [ + 13.1875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.40.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.40.feed_forward.down_proj": { + "inputs": [ + [ + [ + 157.0 + ] + ] + ], + "outputs": [ + [ + [ + 28.5 + ] + ], + [ + [ + 127.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5234375 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 35.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.o_proj": { + "inputs": [ + [ + [ + 12.0625 + ] + ] + ], + "outputs": [ + [ + [ + 160.0 + ] + ], + [ + [ + 177.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.4765625 + ] + ], + [ + [ + 16.875 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 19.875 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4765625 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.9375 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.875 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 19.875 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 17.125 + ] + ], + [ + [ + 15.9375 + ] + ], + [ + [ + 19.875 + ] + ] + ], + "outputs": [ + [ + [ + 10.9375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.41.feed_forward.router": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "outputs": [ + [ + [ + 37.25 + ] + ], + [ + [ + 111.0 + ] + ], + [ + [ + 98.5 + ] + ], + [ + [ + 171.0 + ] + ], + [ + [ + 146.0 + ] + ], + [ + [ + 107.0 + ] + ], + [ + [ + 150.0 + ] + ], + [ + [ + 126.5 + ] + ], + [ + [ + 115.0 + ] + ], + [ + [ + 163.0 + ] + ], + [ + [ + 117.0 + ] + ], + [ + [ + 146.0 + ] + ], + [ + [ + 268.0 + ] + ], + [ + [ + 139.0 + ] + ], + [ + [ + 248.0 + ] + ], + [ + [ + 126.5 + ] + ], + [ + [ + 214.0 + ] + ] + ] + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.416015625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28515625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.369140625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.283203125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.412109375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.359375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 200.0 + ] + ] + ], + "outputs": [ + [ + [ + 39.0 + ] + ], + [ + [ + 4.286760286406354e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6640625 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 39.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.o_proj": { + "inputs": [ + [ + [ + 15.0625 + ] + ] + ], + "outputs": [ + [ + [ + 65.0 + ] + ], + [ + [ + 146.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.447265625 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.3125 + ] + ], + [ + [ + 26.5 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 16.5 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.3125 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.9375 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 26.5 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 16.5 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.8125 + ] + ], + [ + [ + 26.5 + ] + ], + [ + [ + 16.5 + ] + ] + ], + "outputs": [ + [ + [ + 15.0625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.42.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 43.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.44140625 + ] + ] + } + }, + "language_model.model.layers.42.feed_forward.down_proj": { + "inputs": [ + [ + [ + 264.0 + ] + ] + ], + "outputs": [ + [ + [ + 70.5 + ] + ], + [ + [ + 218.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.796875 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 43.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.o_proj": { + "inputs": [ + [ + [ + 11.75 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 232.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.82421875 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2109375 + ] + ], + [ + [ + 15.8125 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.75 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2109375 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.0 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.8125 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.75 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 17.5 + ] + ], + [ + [ + 15.8125 + ] + ], + [ + [ + 11.75 + ] + ] + ], + "outputs": [ + [ + [ + 11.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.43.feed_forward.router": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "outputs": [ + [ + [ + 113.5 + ] + ], + [ + [ + 251.0 + ] + ], + [ + [ + 223.0 + ] + ], + [ + [ + 262.0 + ] + ], + [ + [ + 85.0 + ] + ], + [ + [ + 336.0 + ] + ], + [ + [ + 85.0 + ] + ], + [ + [ + 208.0 + ] + ], + [ + [ + 200.0 + ] + ], + [ + [ + 127.0 + ] + ], + [ + [ + 159.0 + ] + ], + [ + [ + 133.0 + ] + ], + [ + [ + 182.0 + ] + ], + [ + [ + 264.0 + ] + ], + [ + [ + 156.0 + ] + ], + [ + [ + 228.0 + ] + ], + [ + [ + 215.0 + ] + ] + ] + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.42578125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11572265625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3203125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4140625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.455078125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.41796875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1044921875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4609375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34765625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 188.0 + ] + ] + ], + "outputs": [ + [ + [ + 110.0 + ] + ], + [ + [ + 3.705223038250453e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.72265625 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 29.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.o_proj": { + "inputs": [ + [ + [ + 16.375 + ] + ] + ], + "outputs": [ + [ + [ + 81.0 + ] + ], + [ + [ + 170.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.671875 + ] + ], + [ + [ + 17.25 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.671875 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 15.6875 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.25 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 21.375 + ] + ], + [ + [ + 17.25 + ] + ], + [ + [ + 17.25 + ] + ] + ], + "outputs": [ + [ + [ + 16.375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.44.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 42.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.44.feed_forward.down_proj": { + "inputs": [ + [ + [ + 272.0 + ] + ] + ], + "outputs": [ + [ + [ + 75.5 + ] + ], + [ + [ + 200.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.76171875 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 28.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.o_proj": { + "inputs": [ + [ + [ + 21.625 + ] + ] + ], + "outputs": [ + [ + [ + 170.0 + ] + ], + [ + [ + 258.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.765625 + ] + ], + [ + [ + 19.75 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 22.0 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.765625 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.8125 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.75 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 22.0 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 21.5 + ] + ], + [ + [ + 19.75 + ] + ], + [ + [ + 22.0 + ] + ] + ], + "outputs": [ + [ + [ + 21.625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.45.feed_forward.router": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "outputs": [ + [ + [ + 23.625 + ] + ], + [ + [ + 198.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 78.5 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 30.25 + ] + ] + ] + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.376953125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.091796875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1025390625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 434.0 + ] + ] + ], + "outputs": [ + [ + [ + 60.0 + ] + ], + [ + [ + 4.286760286406354e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.447265625 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 27.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.o_proj": { + "inputs": [ + [ + [ + 16.375 + ] + ] + ], + "outputs": [ + [ + [ + 126.0 + ] + ], + [ + [ + 221.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.8203125 + ] + ], + [ + [ + 18.25 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 16.75 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.8203125 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.8125 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.25 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 16.75 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 23.0 + ] + ], + [ + [ + 18.25 + ] + ], + [ + [ + 16.75 + ] + ] + ], + "outputs": [ + [ + [ + 16.375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.46.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 40.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.46.feed_forward.down_proj": { + "inputs": [ + [ + [ + 450.0 + ] + ] + ], + "outputs": [ + [ + [ + 1680.0 + ] + ], + [ + [ + 2400.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5703125 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.o_proj": { + "inputs": [ + [ + [ + 10.0625 + ] + ] + ], + "outputs": [ + [ + [ + 96.5 + ] + ], + [ + [ + 125.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.419921875 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.53125 + ] + ], + [ + [ + 16.5 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.75 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.53125 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.4375 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.5 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 12.75 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 6.6875 + ] + ], + [ + [ + 16.5 + ] + ], + [ + [ + 12.75 + ] + ] + ], + "outputs": [ + [ + [ + 10.0625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.47.feed_forward.router": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "outputs": [ + [ + [ + 486.0 + ] + ], + [ + [ + 150.0 + ] + ], + [ + [ + 53.75 + ] + ], + [ + [ + 21.375 + ] + ], + [ + [ + 189.0 + ] + ], + [ + [ + 131.0 + ] + ], + [ + [ + 536.0 + ] + ], + [ + [ + 288.0 + ] + ], + [ + [ + 43.0 + ] + ], + [ + [ + 78.0 + ] + ], + [ + [ + 76.5 + ] + ], + [ + [ + 223.0 + ] + ], + [ + [ + 57.75 + ] + ], + [ + [ + 236.0 + ] + ], + [ + [ + 158.0 + ] + ], + [ + [ + 241.0 + ] + ], + [ + [ + 207.0 + ] + ] + ] + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.287109375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.373046875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11572265625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11767578125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7578125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.74609375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.82421875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.478515625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 1.015625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 1.0546875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.54296875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4921875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7578125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.84375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.77734375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.46875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 860.0 + ] + ] + ], + "outputs": [ + [ + [ + 700.0 + ] + ], + [ + [ + 3.705223038250453e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.419921875 + ] + ] + } + }, + "language_model.lm_head": { + "inputs": [ + [ + [ + 54.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.984375 + ] + ] + } + } + } +} \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_4_8.npz b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_4_8.npz new file mode 100644 index 000000000000..0d045f5fbdfa Binary files /dev/null and b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_4_8.npz differ diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_4_8_mod_list.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_4_8_mod_list.json new file mode 100644 index 000000000000..449fce78b77d --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_4_8_mod_list.json @@ -0,0 +1,1559 @@ +[ + "vision_model.patch_embedding.linear", + "vision_model.model.layers.0.self_attn.qkv_proj", + "vision_model.model.layers.0.self_attn.o_proj", + "vision_model.model.layers.0.mlp.fc1", + "vision_model.model.layers.0.mlp.fc2", + "vision_model.model.layers.1.self_attn.qkv_proj", + "vision_model.model.layers.1.self_attn.o_proj", + "vision_model.model.layers.1.mlp.fc1", + "vision_model.model.layers.1.mlp.fc2", + "vision_model.model.layers.2.self_attn.qkv_proj", + "vision_model.model.layers.2.self_attn.o_proj", + "vision_model.model.layers.2.mlp.fc1", + "vision_model.model.layers.2.mlp.fc2", + "vision_model.model.layers.3.self_attn.qkv_proj", + "vision_model.model.layers.3.self_attn.o_proj", + "vision_model.model.layers.3.mlp.fc1", + "vision_model.model.layers.3.mlp.fc2", + "vision_model.model.layers.4.self_attn.qkv_proj", + "vision_model.model.layers.4.self_attn.o_proj", + "vision_model.model.layers.4.mlp.fc1", + "vision_model.model.layers.4.mlp.fc2", + "vision_model.model.layers.5.self_attn.qkv_proj", + "vision_model.model.layers.5.self_attn.o_proj", + "vision_model.model.layers.5.mlp.fc1", + "vision_model.model.layers.5.mlp.fc2", + "vision_model.model.layers.6.self_attn.qkv_proj", + "vision_model.model.layers.6.self_attn.o_proj", + "vision_model.model.layers.6.mlp.fc1", + "vision_model.model.layers.6.mlp.fc2", + "vision_model.model.layers.7.self_attn.qkv_proj", + "vision_model.model.layers.7.self_attn.o_proj", + "vision_model.model.layers.7.mlp.fc1", + "vision_model.model.layers.7.mlp.fc2", + "vision_model.model.layers.8.self_attn.qkv_proj", + "vision_model.model.layers.8.self_attn.o_proj", + "vision_model.model.layers.8.mlp.fc1", + "vision_model.model.layers.8.mlp.fc2", + "vision_model.model.layers.9.self_attn.qkv_proj", + "vision_model.model.layers.9.self_attn.o_proj", + "vision_model.model.layers.9.mlp.fc1", + "vision_model.model.layers.9.mlp.fc2", + "vision_model.model.layers.10.self_attn.qkv_proj", + "vision_model.model.layers.10.self_attn.o_proj", + "vision_model.model.layers.10.mlp.fc1", + "vision_model.model.layers.10.mlp.fc2", + "vision_model.model.layers.11.self_attn.qkv_proj", + "vision_model.model.layers.11.self_attn.o_proj", + "vision_model.model.layers.11.mlp.fc1", + "vision_model.model.layers.11.mlp.fc2", + "vision_model.model.layers.12.self_attn.qkv_proj", + "vision_model.model.layers.12.self_attn.o_proj", + "vision_model.model.layers.12.mlp.fc1", + "vision_model.model.layers.12.mlp.fc2", + "vision_model.model.layers.13.self_attn.qkv_proj", + "vision_model.model.layers.13.self_attn.o_proj", + "vision_model.model.layers.13.mlp.fc1", + "vision_model.model.layers.13.mlp.fc2", + "vision_model.model.layers.14.self_attn.qkv_proj", + "vision_model.model.layers.14.self_attn.o_proj", + "vision_model.model.layers.14.mlp.fc1", + "vision_model.model.layers.14.mlp.fc2", + "vision_model.model.layers.15.self_attn.qkv_proj", + "vision_model.model.layers.15.self_attn.o_proj", + "vision_model.model.layers.15.mlp.fc1", + "vision_model.model.layers.15.mlp.fc2", + "vision_model.model.layers.16.self_attn.qkv_proj", + "vision_model.model.layers.16.self_attn.o_proj", + "vision_model.model.layers.16.mlp.fc1", + "vision_model.model.layers.16.mlp.fc2", + "vision_model.model.layers.17.self_attn.qkv_proj", + "vision_model.model.layers.17.self_attn.o_proj", + "vision_model.model.layers.17.mlp.fc1", + "vision_model.model.layers.17.mlp.fc2", + "vision_model.model.layers.18.self_attn.qkv_proj", + "vision_model.model.layers.18.self_attn.o_proj", + "vision_model.model.layers.18.mlp.fc1", + "vision_model.model.layers.18.mlp.fc2", + "vision_model.model.layers.19.self_attn.qkv_proj", + "vision_model.model.layers.19.self_attn.o_proj", + "vision_model.model.layers.19.mlp.fc1", + "vision_model.model.layers.19.mlp.fc2", + "vision_model.model.layers.20.self_attn.qkv_proj", + "vision_model.model.layers.20.self_attn.o_proj", + "vision_model.model.layers.20.mlp.fc1", + "vision_model.model.layers.20.mlp.fc2", + "vision_model.model.layers.21.self_attn.qkv_proj", + "vision_model.model.layers.21.self_attn.o_proj", + "vision_model.model.layers.21.mlp.fc1", + "vision_model.model.layers.21.mlp.fc2", + "vision_model.model.layers.22.self_attn.qkv_proj", + "vision_model.model.layers.22.self_attn.o_proj", + "vision_model.model.layers.22.mlp.fc1", + "vision_model.model.layers.22.mlp.fc2", + "vision_model.model.layers.23.self_attn.qkv_proj", + "vision_model.model.layers.23.self_attn.o_proj", + "vision_model.model.layers.23.mlp.fc1", + "vision_model.model.layers.23.mlp.fc2", + "vision_model.model.layers.24.self_attn.qkv_proj", + "vision_model.model.layers.24.self_attn.o_proj", + "vision_model.model.layers.24.mlp.fc1", + "vision_model.model.layers.24.mlp.fc2", + "vision_model.model.layers.25.self_attn.qkv_proj", + "vision_model.model.layers.25.self_attn.o_proj", + "vision_model.model.layers.25.mlp.fc1", + "vision_model.model.layers.25.mlp.fc2", + "vision_model.model.layers.26.self_attn.qkv_proj", + "vision_model.model.layers.26.self_attn.o_proj", + "vision_model.model.layers.26.mlp.fc1", + "vision_model.model.layers.26.mlp.fc2", + "vision_model.model.layers.27.self_attn.qkv_proj", + "vision_model.model.layers.27.self_attn.o_proj", + "vision_model.model.layers.27.mlp.fc1", + "vision_model.model.layers.27.mlp.fc2", + "vision_model.model.layers.28.self_attn.qkv_proj", + "vision_model.model.layers.28.self_attn.o_proj", + "vision_model.model.layers.28.mlp.fc1", + "vision_model.model.layers.28.mlp.fc2", + "vision_model.model.layers.29.self_attn.qkv_proj", + "vision_model.model.layers.29.self_attn.o_proj", + "vision_model.model.layers.29.mlp.fc1", + "vision_model.model.layers.29.mlp.fc2", + "vision_model.model.layers.30.self_attn.qkv_proj", + "vision_model.model.layers.30.self_attn.o_proj", + "vision_model.model.layers.30.mlp.fc1", + "vision_model.model.layers.30.mlp.fc2", + "vision_model.model.layers.31.self_attn.qkv_proj", + "vision_model.model.layers.31.self_attn.o_proj", + "vision_model.model.layers.31.mlp.fc1", + "vision_model.model.layers.31.mlp.fc2", + "vision_model.model.layers.32.self_attn.qkv_proj", + "vision_model.model.layers.32.self_attn.o_proj", + "vision_model.model.layers.32.mlp.fc1", + "vision_model.model.layers.32.mlp.fc2", + "vision_model.model.layers.33.self_attn.qkv_proj", + "vision_model.model.layers.33.self_attn.o_proj", + "vision_model.model.layers.33.mlp.fc1", + "vision_model.model.layers.33.mlp.fc2", + "vision_model.vision_adapter.mlp.fc1", + "vision_model.vision_adapter.mlp.fc2", + "multi_modal_projector.linear_1", + "language_model.model.layers.0.self_attn.qkv_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.0.self_attn.attn.impl.softmax", + "language_model.model.layers.0.self_attn.attn.impl.matmul_av", + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.0.self_attn.attn.impl.k_cache", + "language_model.model.layers.0.self_attn.attn.impl.v_cache", + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.0.feed_forward.gate_up_proj", + "language_model.model.layers.0.feed_forward.down_proj", + "language_model.model.layers.1.self_attn.qkv_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.1.self_attn.attn.impl.softmax", + "language_model.model.layers.1.self_attn.attn.impl.matmul_av", + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.1.self_attn.attn.impl.k_cache", + "language_model.model.layers.1.self_attn.attn.impl.v_cache", + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.1.feed_forward.router", + "language_model.model.layers.1.feed_forward.experts", + "language_model.model.layers.1.feed_forward.experts.moe_op", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.1.feed_forward.shared_expert.down_proj", + "language_model.model.layers.2.self_attn.qkv_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.2.self_attn.attn.impl.softmax", + "language_model.model.layers.2.self_attn.attn.impl.matmul_av", + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.2.self_attn.attn.impl.k_cache", + "language_model.model.layers.2.self_attn.attn.impl.v_cache", + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.2.feed_forward.gate_up_proj", + "language_model.model.layers.2.feed_forward.down_proj", + "language_model.model.layers.3.self_attn.qkv_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.3.self_attn.attn.impl.softmax", + "language_model.model.layers.3.self_attn.attn.impl.matmul_av", + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.3.self_attn.attn.impl.k_cache", + "language_model.model.layers.3.self_attn.attn.impl.v_cache", + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.3.feed_forward.router", + "language_model.model.layers.3.feed_forward.experts", + "language_model.model.layers.3.feed_forward.experts.moe_op", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.3.feed_forward.shared_expert.down_proj", + "language_model.model.layers.4.self_attn.qkv_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.4.self_attn.attn.impl.softmax", + "language_model.model.layers.4.self_attn.attn.impl.matmul_av", + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.4.self_attn.attn.impl.k_cache", + "language_model.model.layers.4.self_attn.attn.impl.v_cache", + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.4.feed_forward.gate_up_proj", + "language_model.model.layers.4.feed_forward.down_proj", + "language_model.model.layers.5.self_attn.qkv_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.5.self_attn.attn.impl.softmax", + "language_model.model.layers.5.self_attn.attn.impl.matmul_av", + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.5.self_attn.attn.impl.k_cache", + "language_model.model.layers.5.self_attn.attn.impl.v_cache", + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.5.feed_forward.router", + "language_model.model.layers.5.feed_forward.experts", + "language_model.model.layers.5.feed_forward.experts.moe_op", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.5.feed_forward.shared_expert.down_proj", + "language_model.model.layers.6.self_attn.qkv_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.6.self_attn.attn.impl.softmax", + "language_model.model.layers.6.self_attn.attn.impl.matmul_av", + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.6.self_attn.attn.impl.k_cache", + "language_model.model.layers.6.self_attn.attn.impl.v_cache", + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.6.feed_forward.gate_up_proj", + "language_model.model.layers.6.feed_forward.down_proj", + "language_model.model.layers.7.self_attn.qkv_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.7.self_attn.attn.impl.softmax", + "language_model.model.layers.7.self_attn.attn.impl.matmul_av", + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.7.self_attn.attn.impl.k_cache", + "language_model.model.layers.7.self_attn.attn.impl.v_cache", + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.7.feed_forward.router", + "language_model.model.layers.7.feed_forward.experts", + "language_model.model.layers.7.feed_forward.experts.moe_op", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.7.feed_forward.shared_expert.down_proj", + "language_model.model.layers.8.self_attn.qkv_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.8.self_attn.attn.impl.softmax", + "language_model.model.layers.8.self_attn.attn.impl.matmul_av", + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.8.self_attn.attn.impl.k_cache", + "language_model.model.layers.8.self_attn.attn.impl.v_cache", + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.8.feed_forward.gate_up_proj", + "language_model.model.layers.8.feed_forward.down_proj", + "language_model.model.layers.9.self_attn.qkv_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.9.self_attn.attn.impl.softmax", + "language_model.model.layers.9.self_attn.attn.impl.matmul_av", + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.9.self_attn.attn.impl.k_cache", + "language_model.model.layers.9.self_attn.attn.impl.v_cache", + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.9.feed_forward.router", + "language_model.model.layers.9.feed_forward.experts", + "language_model.model.layers.9.feed_forward.experts.moe_op", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.9.feed_forward.shared_expert.down_proj", + "language_model.model.layers.10.self_attn.qkv_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.10.self_attn.attn.impl.softmax", + "language_model.model.layers.10.self_attn.attn.impl.matmul_av", + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.10.self_attn.attn.impl.k_cache", + "language_model.model.layers.10.self_attn.attn.impl.v_cache", + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.10.feed_forward.gate_up_proj", + "language_model.model.layers.10.feed_forward.down_proj", + "language_model.model.layers.11.self_attn.qkv_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.11.self_attn.attn.impl.softmax", + "language_model.model.layers.11.self_attn.attn.impl.matmul_av", + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.11.self_attn.attn.impl.k_cache", + "language_model.model.layers.11.self_attn.attn.impl.v_cache", + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.11.feed_forward.router", + "language_model.model.layers.11.feed_forward.experts", + "language_model.model.layers.11.feed_forward.experts.moe_op", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.11.feed_forward.shared_expert.down_proj", + "language_model.model.layers.12.self_attn.qkv_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.12.self_attn.attn.impl.softmax", + "language_model.model.layers.12.self_attn.attn.impl.matmul_av", + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.12.self_attn.attn.impl.k_cache", + "language_model.model.layers.12.self_attn.attn.impl.v_cache", + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.12.feed_forward.gate_up_proj", + "language_model.model.layers.12.feed_forward.down_proj", + "language_model.model.layers.13.self_attn.qkv_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.13.self_attn.attn.impl.softmax", + "language_model.model.layers.13.self_attn.attn.impl.matmul_av", + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.13.self_attn.attn.impl.k_cache", + "language_model.model.layers.13.self_attn.attn.impl.v_cache", + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.13.feed_forward.router", + "language_model.model.layers.13.feed_forward.experts", + "language_model.model.layers.13.feed_forward.experts.moe_op", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.13.feed_forward.shared_expert.down_proj", + "language_model.model.layers.14.self_attn.qkv_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.14.self_attn.attn.impl.softmax", + "language_model.model.layers.14.self_attn.attn.impl.matmul_av", + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.14.self_attn.attn.impl.k_cache", + "language_model.model.layers.14.self_attn.attn.impl.v_cache", + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.14.feed_forward.gate_up_proj", + "language_model.model.layers.14.feed_forward.down_proj", + "language_model.model.layers.15.self_attn.qkv_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.15.self_attn.attn.impl.softmax", + "language_model.model.layers.15.self_attn.attn.impl.matmul_av", + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.15.self_attn.attn.impl.k_cache", + "language_model.model.layers.15.self_attn.attn.impl.v_cache", + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.15.feed_forward.router", + "language_model.model.layers.15.feed_forward.experts", + "language_model.model.layers.15.feed_forward.experts.moe_op", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.15.feed_forward.shared_expert.down_proj", + "language_model.model.layers.16.self_attn.qkv_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.16.self_attn.attn.impl.softmax", + "language_model.model.layers.16.self_attn.attn.impl.matmul_av", + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.16.self_attn.attn.impl.k_cache", + "language_model.model.layers.16.self_attn.attn.impl.v_cache", + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.16.feed_forward.gate_up_proj", + "language_model.model.layers.16.feed_forward.down_proj", + "language_model.model.layers.17.self_attn.qkv_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.17.self_attn.attn.impl.softmax", + "language_model.model.layers.17.self_attn.attn.impl.matmul_av", + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.17.self_attn.attn.impl.k_cache", + "language_model.model.layers.17.self_attn.attn.impl.v_cache", + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.17.feed_forward.router", + "language_model.model.layers.17.feed_forward.experts", + "language_model.model.layers.17.feed_forward.experts.moe_op", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.17.feed_forward.shared_expert.down_proj", + "language_model.model.layers.18.self_attn.qkv_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.18.self_attn.attn.impl.softmax", + "language_model.model.layers.18.self_attn.attn.impl.matmul_av", + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.18.self_attn.attn.impl.k_cache", + "language_model.model.layers.18.self_attn.attn.impl.v_cache", + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.18.feed_forward.gate_up_proj", + "language_model.model.layers.18.feed_forward.down_proj", + "language_model.model.layers.19.self_attn.qkv_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.19.self_attn.attn.impl.softmax", + "language_model.model.layers.19.self_attn.attn.impl.matmul_av", + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.19.self_attn.attn.impl.k_cache", + "language_model.model.layers.19.self_attn.attn.impl.v_cache", + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.19.feed_forward.router", + "language_model.model.layers.19.feed_forward.experts", + "language_model.model.layers.19.feed_forward.experts.moe_op", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.19.feed_forward.shared_expert.down_proj", + "language_model.model.layers.20.self_attn.qkv_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.20.self_attn.attn.impl.softmax", + "language_model.model.layers.20.self_attn.attn.impl.matmul_av", + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.20.self_attn.attn.impl.k_cache", + "language_model.model.layers.20.self_attn.attn.impl.v_cache", + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.20.feed_forward.gate_up_proj", + "language_model.model.layers.20.feed_forward.down_proj", + "language_model.model.layers.21.self_attn.qkv_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.21.self_attn.attn.impl.softmax", + "language_model.model.layers.21.self_attn.attn.impl.matmul_av", + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.21.self_attn.attn.impl.k_cache", + "language_model.model.layers.21.self_attn.attn.impl.v_cache", + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.21.feed_forward.router", + "language_model.model.layers.21.feed_forward.experts", + "language_model.model.layers.21.feed_forward.experts.moe_op", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.21.feed_forward.shared_expert.down_proj", + "language_model.model.layers.22.self_attn.qkv_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.22.self_attn.attn.impl.softmax", + "language_model.model.layers.22.self_attn.attn.impl.matmul_av", + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.22.self_attn.attn.impl.k_cache", + "language_model.model.layers.22.self_attn.attn.impl.v_cache", + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.22.feed_forward.gate_up_proj", + "language_model.model.layers.22.feed_forward.down_proj", + "language_model.model.layers.23.self_attn.qkv_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.23.self_attn.attn.impl.softmax", + "language_model.model.layers.23.self_attn.attn.impl.matmul_av", + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.23.self_attn.attn.impl.k_cache", + "language_model.model.layers.23.self_attn.attn.impl.v_cache", + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.23.feed_forward.router", + "language_model.model.layers.23.feed_forward.experts", + "language_model.model.layers.23.feed_forward.experts.moe_op", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.23.feed_forward.shared_expert.down_proj", + "language_model.model.layers.24.self_attn.qkv_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.24.self_attn.attn.impl.softmax", + "language_model.model.layers.24.self_attn.attn.impl.matmul_av", + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.24.self_attn.attn.impl.k_cache", + "language_model.model.layers.24.self_attn.attn.impl.v_cache", + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.24.feed_forward.gate_up_proj", + "language_model.model.layers.24.feed_forward.down_proj", + "language_model.model.layers.25.self_attn.qkv_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.25.self_attn.attn.impl.softmax", + "language_model.model.layers.25.self_attn.attn.impl.matmul_av", + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.25.self_attn.attn.impl.k_cache", + "language_model.model.layers.25.self_attn.attn.impl.v_cache", + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.25.feed_forward.router", + "language_model.model.layers.25.feed_forward.experts", + "language_model.model.layers.25.feed_forward.experts.moe_op", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.25.feed_forward.shared_expert.down_proj", + "language_model.model.layers.26.self_attn.qkv_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.26.self_attn.attn.impl.softmax", + "language_model.model.layers.26.self_attn.attn.impl.matmul_av", + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.26.self_attn.attn.impl.k_cache", + "language_model.model.layers.26.self_attn.attn.impl.v_cache", + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.26.feed_forward.gate_up_proj", + "language_model.model.layers.26.feed_forward.down_proj", + "language_model.model.layers.27.self_attn.qkv_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.27.self_attn.attn.impl.softmax", + "language_model.model.layers.27.self_attn.attn.impl.matmul_av", + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.27.self_attn.attn.impl.k_cache", + "language_model.model.layers.27.self_attn.attn.impl.v_cache", + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.27.feed_forward.router", + "language_model.model.layers.27.feed_forward.experts", + "language_model.model.layers.27.feed_forward.experts.moe_op", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.27.feed_forward.shared_expert.down_proj", + "language_model.model.layers.28.self_attn.qkv_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.28.self_attn.attn.impl.softmax", + "language_model.model.layers.28.self_attn.attn.impl.matmul_av", + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.28.self_attn.attn.impl.k_cache", + "language_model.model.layers.28.self_attn.attn.impl.v_cache", + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.28.feed_forward.gate_up_proj", + "language_model.model.layers.28.feed_forward.down_proj", + "language_model.model.layers.29.self_attn.qkv_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.29.self_attn.attn.impl.softmax", + "language_model.model.layers.29.self_attn.attn.impl.matmul_av", + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.29.self_attn.attn.impl.k_cache", + "language_model.model.layers.29.self_attn.attn.impl.v_cache", + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.29.feed_forward.router", + "language_model.model.layers.29.feed_forward.experts", + "language_model.model.layers.29.feed_forward.experts.moe_op", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.29.feed_forward.shared_expert.down_proj", + "language_model.model.layers.30.self_attn.qkv_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.30.self_attn.attn.impl.softmax", + "language_model.model.layers.30.self_attn.attn.impl.matmul_av", + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.30.self_attn.attn.impl.k_cache", + "language_model.model.layers.30.self_attn.attn.impl.v_cache", + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.30.feed_forward.gate_up_proj", + "language_model.model.layers.30.feed_forward.down_proj", + "language_model.model.layers.31.self_attn.qkv_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.31.self_attn.attn.impl.softmax", + "language_model.model.layers.31.self_attn.attn.impl.matmul_av", + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.31.self_attn.attn.impl.k_cache", + "language_model.model.layers.31.self_attn.attn.impl.v_cache", + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.31.feed_forward.router", + "language_model.model.layers.31.feed_forward.experts", + "language_model.model.layers.31.feed_forward.experts.moe_op", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.31.feed_forward.shared_expert.down_proj", + "language_model.model.layers.32.self_attn.qkv_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.32.self_attn.attn.impl.softmax", + "language_model.model.layers.32.self_attn.attn.impl.matmul_av", + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.32.self_attn.attn.impl.k_cache", + "language_model.model.layers.32.self_attn.attn.impl.v_cache", + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.32.feed_forward.gate_up_proj", + "language_model.model.layers.32.feed_forward.down_proj", + "language_model.model.layers.33.self_attn.qkv_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.33.self_attn.attn.impl.softmax", + "language_model.model.layers.33.self_attn.attn.impl.matmul_av", + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.33.self_attn.attn.impl.k_cache", + "language_model.model.layers.33.self_attn.attn.impl.v_cache", + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.33.feed_forward.router", + "language_model.model.layers.33.feed_forward.experts", + "language_model.model.layers.33.feed_forward.experts.moe_op", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.33.feed_forward.shared_expert.down_proj", + "language_model.model.layers.34.self_attn.qkv_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.34.self_attn.attn.impl.softmax", + "language_model.model.layers.34.self_attn.attn.impl.matmul_av", + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.34.self_attn.attn.impl.k_cache", + "language_model.model.layers.34.self_attn.attn.impl.v_cache", + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.34.feed_forward.gate_up_proj", + "language_model.model.layers.34.feed_forward.down_proj", + "language_model.model.layers.35.self_attn.qkv_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.35.self_attn.attn.impl.softmax", + "language_model.model.layers.35.self_attn.attn.impl.matmul_av", + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.35.self_attn.attn.impl.k_cache", + "language_model.model.layers.35.self_attn.attn.impl.v_cache", + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.35.feed_forward.router", + "language_model.model.layers.35.feed_forward.experts", + "language_model.model.layers.35.feed_forward.experts.moe_op", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.35.feed_forward.shared_expert.down_proj", + "language_model.model.layers.36.self_attn.qkv_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.36.self_attn.attn.impl.softmax", + "language_model.model.layers.36.self_attn.attn.impl.matmul_av", + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.36.self_attn.attn.impl.k_cache", + "language_model.model.layers.36.self_attn.attn.impl.v_cache", + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.36.feed_forward.gate_up_proj", + "language_model.model.layers.36.feed_forward.down_proj", + "language_model.model.layers.37.self_attn.qkv_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.37.self_attn.attn.impl.softmax", + "language_model.model.layers.37.self_attn.attn.impl.matmul_av", + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.37.self_attn.attn.impl.k_cache", + "language_model.model.layers.37.self_attn.attn.impl.v_cache", + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.37.feed_forward.router", + "language_model.model.layers.37.feed_forward.experts", + "language_model.model.layers.37.feed_forward.experts.moe_op", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.37.feed_forward.shared_expert.down_proj", + "language_model.model.layers.38.self_attn.qkv_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.38.self_attn.attn.impl.softmax", + "language_model.model.layers.38.self_attn.attn.impl.matmul_av", + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.38.self_attn.attn.impl.k_cache", + "language_model.model.layers.38.self_attn.attn.impl.v_cache", + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.38.feed_forward.gate_up_proj", + "language_model.model.layers.38.feed_forward.down_proj", + "language_model.model.layers.39.self_attn.qkv_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.39.self_attn.attn.impl.softmax", + "language_model.model.layers.39.self_attn.attn.impl.matmul_av", + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.39.self_attn.attn.impl.k_cache", + "language_model.model.layers.39.self_attn.attn.impl.v_cache", + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.39.feed_forward.router", + "language_model.model.layers.39.feed_forward.experts", + "language_model.model.layers.39.feed_forward.experts.moe_op", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.39.feed_forward.shared_expert.down_proj", + "language_model.model.layers.40.self_attn.qkv_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.40.self_attn.attn.impl.softmax", + "language_model.model.layers.40.self_attn.attn.impl.matmul_av", + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.40.self_attn.attn.impl.k_cache", + "language_model.model.layers.40.self_attn.attn.impl.v_cache", + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.40.feed_forward.gate_up_proj", + "language_model.model.layers.40.feed_forward.down_proj", + "language_model.model.layers.41.self_attn.qkv_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.41.self_attn.attn.impl.softmax", + "language_model.model.layers.41.self_attn.attn.impl.matmul_av", + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.41.self_attn.attn.impl.k_cache", + "language_model.model.layers.41.self_attn.attn.impl.v_cache", + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.41.feed_forward.router", + "language_model.model.layers.41.feed_forward.experts", + "language_model.model.layers.41.feed_forward.experts.moe_op", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.41.feed_forward.shared_expert.down_proj", + "language_model.model.layers.42.self_attn.qkv_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.42.self_attn.attn.impl.softmax", + "language_model.model.layers.42.self_attn.attn.impl.matmul_av", + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.42.self_attn.attn.impl.k_cache", + "language_model.model.layers.42.self_attn.attn.impl.v_cache", + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.42.feed_forward.gate_up_proj", + "language_model.model.layers.42.feed_forward.down_proj", + "language_model.model.layers.43.self_attn.qkv_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.43.self_attn.attn.impl.softmax", + "language_model.model.layers.43.self_attn.attn.impl.matmul_av", + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.43.self_attn.attn.impl.k_cache", + "language_model.model.layers.43.self_attn.attn.impl.v_cache", + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.43.feed_forward.router", + "language_model.model.layers.43.feed_forward.experts", + "language_model.model.layers.43.feed_forward.experts.moe_op", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.43.feed_forward.shared_expert.down_proj", + "language_model.model.layers.44.self_attn.qkv_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.44.self_attn.attn.impl.softmax", + "language_model.model.layers.44.self_attn.attn.impl.matmul_av", + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.44.self_attn.attn.impl.k_cache", + "language_model.model.layers.44.self_attn.attn.impl.v_cache", + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.44.feed_forward.gate_up_proj", + "language_model.model.layers.44.feed_forward.down_proj", + "language_model.model.layers.45.self_attn.qkv_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.45.self_attn.attn.impl.softmax", + "language_model.model.layers.45.self_attn.attn.impl.matmul_av", + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.45.self_attn.attn.impl.k_cache", + "language_model.model.layers.45.self_attn.attn.impl.v_cache", + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.45.feed_forward.router", + "language_model.model.layers.45.feed_forward.experts", + "language_model.model.layers.45.feed_forward.experts.moe_op", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.45.feed_forward.shared_expert.down_proj", + "language_model.model.layers.46.self_attn.qkv_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.46.self_attn.attn.impl.softmax", + "language_model.model.layers.46.self_attn.attn.impl.matmul_av", + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.46.self_attn.attn.impl.k_cache", + "language_model.model.layers.46.self_attn.attn.impl.v_cache", + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.46.feed_forward.gate_up_proj", + "language_model.model.layers.46.feed_forward.down_proj", + "language_model.model.layers.47.self_attn.qkv_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.47.self_attn.attn.impl.softmax", + "language_model.model.layers.47.self_attn.attn.impl.matmul_av", + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.47.self_attn.attn.impl.k_cache", + "language_model.model.layers.47.self_attn.attn.impl.v_cache", + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.47.feed_forward.router", + "language_model.model.layers.47.feed_forward.experts", + "language_model.model.layers.47.feed_forward.experts.moe_op", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.47.feed_forward.shared_expert.down_proj", + "language_model.lm_head" +] \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_5_8.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_5_8.json new file mode 100644 index 000000000000..b05fefd76f26 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_5_8.json @@ -0,0 +1,24263 @@ +{ + "GlobalRank": null, + "LocalRank": 5, + "Mode": "DynamicRange", + "Nodes": { + "language_model.model.layers.0.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 11.1875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3671875 + ] + ] + } + }, + "language_model.model.layers.0.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.50390625 + ] + ] + ], + "outputs": [ + [ + [ + 0.83203125 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.322265625 + ] + ], + [ + [ + 6.3125 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.61328125 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.322265625 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.5546875 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 6.3125 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.61328125 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 4.3125 + ] + ], + [ + [ + 6.3125 + ] + ], + [ + [ + 0.61328125 + ] + ] + ], + "outputs": [ + [ + [ + 0.50390625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.0.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 2.546875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.0.feed_forward.down_proj": { + "inputs": [ + [ + [ + 6.3125 + ] + ] + ], + "outputs": [ + [ + [ + 26.75 + ] + ], + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7421875 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 10.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.8125 + ] + ] + ], + "outputs": [ + [ + [ + 0.67578125 + ] + ], + [ + [ + 1.3984375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.70703125 + ] + ], + [ + [ + 8.875 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.91015625 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.70703125 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.69140625 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 8.875 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.91015625 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 8.3125 + ] + ], + [ + [ + 8.875 + ] + ], + [ + [ + 0.91015625 + ] + ] + ], + "outputs": [ + [ + [ + 0.8125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.1.feed_forward.router": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "outputs": [ + [ + [ + 1.8125 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 4.03125 + ] + ], + [ + [ + 20.375 + ] + ], + [ + [ + 5.125 + ] + ], + [ + [ + 21.625 + ] + ], + [ + [ + 27.0 + ] + ], + [ + [ + 15.3125 + ] + ], + [ + [ + 13.4375 + ] + ], + [ + [ + 12.0 + ] + ], + [ + [ + 20.875 + ] + ], + [ + [ + 13.75 + ] + ], + [ + [ + 23.5 + ] + ], + [ + [ + 17.875 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 18.25 + ] + ], + [ + [ + 15.6875 + ] + ] + ] + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11572265625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08544921875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.349609375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33203125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.283203125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.322265625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 5.0 + ] + ] + ], + "outputs": [ + [ + [ + 0.455078125 + ] + ], + [ + [ + 4.652297985247206e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 9.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.64453125 + ] + ] + ], + "outputs": [ + [ + [ + 0.7734375 + ] + ], + [ + [ + 1.4921875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.55078125 + ] + ], + [ + [ + 8.9375 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1875 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.55078125 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.7578125 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 8.9375 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.1875 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.3125 + ] + ], + [ + [ + 8.9375 + ] + ], + [ + [ + 1.1875 + ] + ] + ], + "outputs": [ + [ + [ + 0.64453125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.2.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 6.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33984375 + ] + ] + } + }, + "language_model.model.layers.2.feed_forward.down_proj": { + "inputs": [ + [ + [ + 14.375 + ] + ] + ], + "outputs": [ + [ + [ + 4.75 + ] + ], + [ + [ + 25.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.55078125 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 7.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.244140625 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.5703125 + ] + ] + ], + "outputs": [ + [ + [ + 0.77734375 + ] + ], + [ + [ + 2.40625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.408203125 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.1552734375 + ] + ], + [ + [ + 2.328125 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.46484375 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.1552734375 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.46484375 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 3.046875 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.5625 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 1.578125 + ] + ], + [ + [ + 3.09375 + ] + ], + [ + [ + 0.57421875 + ] + ] + ], + "outputs": [ + [ + [ + 0.5703125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.3.feed_forward.router": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.52734375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "outputs": [ + [ + [ + 1568.0 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 20.0 + ] + ], + [ + [ + 173.0 + ] + ], + [ + [ + 10.4375 + ] + ], + [ + [ + 116.0 + ] + ], + [ + [ + 13.625 + ] + ], + [ + [ + 3.796875 + ] + ], + [ + [ + 20.5 + ] + ], + [ + [ + 22.375 + ] + ], + [ + [ + 19.625 + ] + ], + [ + [ + 12.5 + ] + ], + [ + [ + 18.0 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 28.5 + ] + ], + [ + [ + 19.125 + ] + ] + ] + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.76171875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11279296875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.74609375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1025390625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11767578125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08447265625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11181640625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 6.3125 + ] + ] + ], + "outputs": [ + [ + [ + 0.484375 + ] + ], + [ + [ + 7.011677677765431e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 14.3125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.8046875 + ] + ] + ], + "outputs": [ + [ + [ + 1.46875 + ] + ], + [ + [ + 2.328125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.703125 + ] + ], + [ + [ + 12.1875 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.625 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.703125 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.7421875 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 12.1875 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.625 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 8.1875 + ] + ], + [ + [ + 12.1875 + ] + ], + [ + [ + 4.625 + ] + ] + ], + "outputs": [ + [ + [ + 1.8046875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.4.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 6.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.32421875 + ] + ] + } + }, + "language_model.model.layers.4.feed_forward.down_proj": { + "inputs": [ + [ + [ + 9.1875 + ] + ] + ], + "outputs": [ + [ + [ + 2.796875 + ] + ], + [ + [ + 5.1875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 18.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.171875 + ] + ] + ], + "outputs": [ + [ + [ + 1.4921875 + ] + ], + [ + [ + 2.40625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.65625 + ] + ], + [ + [ + 10.75 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.78125 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.65625 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.625 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 10.75 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.78125 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 8.0625 + ] + ], + [ + [ + 10.75 + ] + ], + [ + [ + 3.78125 + ] + ] + ], + "outputs": [ + [ + [ + 2.171875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.5.feed_forward.router": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 2.703125 + ] + ], + [ + [ + 20.625 + ] + ], + [ + [ + 23.125 + ] + ], + [ + [ + 13.6875 + ] + ], + [ + [ + 18.625 + ] + ], + [ + [ + 15.1875 + ] + ], + [ + [ + 22.25 + ] + ], + [ + [ + 23.125 + ] + ], + [ + [ + 28.875 + ] + ], + [ + [ + 24.5 + ] + ], + [ + [ + 34.25 + ] + ], + [ + [ + 25.125 + ] + ], + [ + [ + 32.5 + ] + ], + [ + [ + 32.25 + ] + ], + [ + [ + 22.25 + ] + ], + [ + [ + 20.625 + ] + ], + [ + [ + 15.1875 + ] + ] + ] + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11279296875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.111328125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1044921875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.283203125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0830078125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.103515625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.384765625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 5.53125 + ] + ] + ], + "outputs": [ + [ + [ + 0.63671875 + ] + ], + [ + [ + 1.8609191940988822e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.373046875 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 14.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2353515625 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.6328125 + ] + ] + ], + "outputs": [ + [ + [ + 0.94140625 + ] + ], + [ + [ + 1.96875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.9375 + ] + ], + [ + [ + 16.875 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.25 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.9375 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.609375 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.875 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.25 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.625 + ] + ], + [ + [ + 16.875 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "outputs": [ + [ + [ + 1.6328125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.6.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 7.71875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.6.feed_forward.down_proj": { + "inputs": [ + [ + [ + 7.53125 + ] + ] + ], + "outputs": [ + [ + [ + 2.25 + ] + ], + [ + [ + 11.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.474609375 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 7.59375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.8515625 + ] + ] + ], + "outputs": [ + [ + [ + 2.1875 + ] + ], + [ + [ + 2.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.375 + ] + ], + [ + [ + 3.234375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.484375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.8359375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 3.234375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.484375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 4.40625 + ] + ], + [ + [ + 3.234375 + ] + ], + [ + [ + 2.046875 + ] + ] + ], + "outputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.7.feed_forward.router": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "outputs": [ + [ + [ + 3.875 + ] + ], + [ + [ + 31.75 + ] + ], + [ + [ + 48.25 + ] + ], + [ + [ + 39.25 + ] + ], + [ + [ + 15.6875 + ] + ], + [ + [ + 18.625 + ] + ], + [ + [ + 45.0 + ] + ], + [ + [ + 18.0 + ] + ], + [ + [ + 49.75 + ] + ], + [ + [ + 52.25 + ] + ], + [ + [ + 37.0 + ] + ], + [ + [ + 32.0 + ] + ], + [ + [ + 43.75 + ] + ], + [ + [ + 41.75 + ] + ], + [ + [ + 18.75 + ] + ], + [ + [ + 28.5 + ] + ], + [ + [ + 54.0 + ] + ] + ] + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08349609375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10498046875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08837890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0859375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "outputs": [ + [ + [ + 0.494140625 + ] + ], + [ + [ + 1.8609191940988822e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.40625 + ] + ] + ], + "outputs": [ + [ + [ + 2.265625 + ] + ], + [ + [ + 3.390625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.80078125 + ] + ], + [ + [ + 19.125 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.359375 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.80078125 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.3125 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.125 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.359375 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.75 + ] + ], + [ + [ + 19.125 + ] + ], + [ + [ + 3.359375 + ] + ] + ], + "outputs": [ + [ + [ + 2.40625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.8.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 8.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.8.feed_forward.down_proj": { + "inputs": [ + [ + [ + 11.4375 + ] + ] + ], + "outputs": [ + [ + [ + 2.84375 + ] + ], + [ + [ + 9.8125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5703125 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 16.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.0 + ] + ] + ], + "outputs": [ + [ + [ + 1.609375 + ] + ], + [ + [ + 3.640625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.7734375 + ] + ], + [ + [ + 14.0 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.65625 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.7734375 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.1875 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.0 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.65625 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.5 + ] + ], + [ + [ + 14.0 + ] + ], + [ + [ + 4.65625 + ] + ] + ], + "outputs": [ + [ + [ + 4.0 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.9.feed_forward.router": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "outputs": [ + [ + [ + 4.3125 + ] + ], + [ + [ + 38.75 + ] + ], + [ + [ + 35.0 + ] + ], + [ + [ + 33.75 + ] + ], + [ + [ + 59.0 + ] + ], + [ + [ + 36.75 + ] + ], + [ + [ + 45.25 + ] + ], + [ + [ + 36.25 + ] + ], + [ + [ + 16.5 + ] + ], + [ + [ + 31.375 + ] + ], + [ + [ + 48.25 + ] + ], + [ + [ + 59.25 + ] + ], + [ + [ + 30.875 + ] + ], + [ + [ + 29.0 + ] + ], + [ + [ + 37.0 + ] + ], + [ + [ + 34.5 + ] + ], + [ + [ + 65.0 + ] + ] + ] + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11376953125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08935546875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11181640625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08544921875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1044921875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.111328125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.103515625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 9.875 + ] + ] + ], + "outputs": [ + [ + [ + 1.7890625 + ] + ], + [ + [ + 4.652297985247206e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 17.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.09375 + ] + ] + ], + "outputs": [ + [ + [ + 1.984375 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.94921875 + ] + ], + [ + [ + 14.625 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.375 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.94921875 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.984375 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.625 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.375 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.9375 + ] + ], + [ + [ + 14.625 + ] + ], + [ + [ + 2.375 + ] + ] + ], + "outputs": [ + [ + [ + 2.09375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.10.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 9.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.10.feed_forward.down_proj": { + "inputs": [ + [ + [ + 12.5 + ] + ] + ], + "outputs": [ + [ + [ + 3.40625 + ] + ], + [ + [ + 12.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.48828125 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 15.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.5 + ] + ] + ], + "outputs": [ + [ + [ + 8.625 + ] + ], + [ + [ + 8.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.73828125 + ] + ], + [ + [ + 10.0 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.5 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.73828125 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.046875 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 10.0 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.5 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 8.375 + ] + ], + [ + [ + 10.0 + ] + ], + [ + [ + 3.5 + ] + ] + ], + "outputs": [ + [ + [ + 2.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.11.feed_forward.router": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "outputs": [ + [ + [ + 8.125 + ] + ], + [ + [ + 37.5 + ] + ], + [ + [ + 39.25 + ] + ], + [ + [ + 55.25 + ] + ], + [ + [ + 38.75 + ] + ], + [ + [ + 42.25 + ] + ], + [ + [ + 47.75 + ] + ], + [ + [ + 77.0 + ] + ], + [ + [ + 104.5 + ] + ], + [ + [ + 46.75 + ] + ], + [ + [ + 48.5 + ] + ], + [ + [ + 49.0 + ] + ], + [ + [ + 42.5 + ] + ], + [ + [ + 53.25 + ] + ], + [ + [ + 78.0 + ] + ], + [ + [ + 17.25 + ] + ], + [ + [ + 41.5 + ] + ] + ] + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.35546875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08642578125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1044921875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.408203125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 18.25 + ] + ] + ], + "outputs": [ + [ + [ + 0.9375 + ] + ], + [ + [ + 1.0035671368176115e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 20.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.71875 + ] + ] + ], + "outputs": [ + [ + [ + 1.796875 + ] + ], + [ + [ + 3.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0703125 + ] + ], + [ + [ + 19.75 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.265625 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0703125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.390625 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.75 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.265625 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.8125 + ] + ], + [ + [ + 19.5 + ] + ], + [ + [ + 3.265625 + ] + ] + ], + "outputs": [ + [ + [ + 2.71875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.12.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 10.6875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.12.feed_forward.down_proj": { + "inputs": [ + [ + [ + 14.5 + ] + ] + ], + "outputs": [ + [ + [ + 9.25 + ] + ], + [ + [ + 20.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.60546875 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.890625 + ] + ] + ], + "outputs": [ + [ + [ + 2.015625 + ] + ], + [ + [ + 6.34375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.828125 + ] + ], + [ + [ + 18.0 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.28125 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.828125 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.4375 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.0 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.28125 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.5 + ] + ], + [ + [ + 17.875 + ] + ], + [ + [ + 4.09375 + ] + ] + ], + "outputs": [ + [ + [ + 3.890625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.13.feed_forward.router": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "outputs": [ + [ + [ + 5.6875 + ] + ], + [ + [ + 38.25 + ] + ], + [ + [ + 59.5 + ] + ], + [ + [ + 47.5 + ] + ], + [ + [ + 57.5 + ] + ], + [ + [ + 71.5 + ] + ], + [ + [ + 35.5 + ] + ], + [ + [ + 23.125 + ] + ], + [ + [ + 65.5 + ] + ], + [ + [ + 40.25 + ] + ], + [ + [ + 83.0 + ] + ], + [ + [ + 84.5 + ] + ], + [ + [ + 70.0 + ] + ], + [ + [ + 63.5 + ] + ], + [ + [ + 62.75 + ] + ], + [ + [ + 65.5 + ] + ], + [ + [ + 27.75 + ] + ] + ] + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.091796875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10986328125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.111328125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10693359375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.373046875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11474609375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09326171875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0859375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 16.125 + ] + ] + ], + "outputs": [ + [ + [ + 1.625 + ] + ], + [ + [ + 1.8609191940988822e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.310546875 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 20.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.5625 + ] + ] + ], + "outputs": [ + [ + [ + 3.546875 + ] + ], + [ + [ + 5.71875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.85546875 + ] + ], + [ + [ + 15.3125 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.34375 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.85546875 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.9921875 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.3125 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.34375 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.9375 + ] + ], + [ + [ + 15.0625 + ] + ], + [ + [ + 4.34375 + ] + ] + ], + "outputs": [ + [ + [ + 2.5625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.14.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 12.0625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.14.feed_forward.down_proj": { + "inputs": [ + [ + [ + 17.0 + ] + ] + ], + "outputs": [ + [ + [ + 5.375 + ] + ], + [ + [ + 20.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.51171875 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 23.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.21875 + ] + ] + ], + "outputs": [ + [ + [ + 20.0 + ] + ], + [ + [ + 21.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.84765625 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.72265625 + ] + ], + [ + [ + 10.625 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.34375 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.72265625 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.015625 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 10.625 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.34375 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.90625 + ] + ], + [ + [ + 10.625 + ] + ], + [ + [ + 4.34375 + ] + ] + ], + "outputs": [ + [ + [ + 3.21875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.15.feed_forward.router": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 7.03125 + ] + ], + [ + [ + 81.0 + ] + ], + [ + [ + 53.75 + ] + ], + [ + [ + 33.5 + ] + ], + [ + [ + 70.5 + ] + ], + [ + [ + 19.375 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 40.75 + ] + ], + [ + [ + 42.0 + ] + ], + [ + [ + 43.75 + ] + ], + [ + [ + 56.25 + ] + ], + [ + [ + 83.0 + ] + ], + [ + [ + 57.5 + ] + ], + [ + [ + 93.0 + ] + ], + [ + [ + 67.0 + ] + ], + [ + [ + 59.25 + ] + ], + [ + [ + 76.5 + ] + ] + ] + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28515625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1083984375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.240234375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 20.375 + ] + ] + ], + "outputs": [ + [ + [ + 8.8125 + ] + ], + [ + [ + 4.652297985247206e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.76171875 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 23.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.828125 + ] + ] + ], + "outputs": [ + [ + [ + 2.4375 + ] + ], + [ + [ + 6.3125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1796875 + ] + ], + [ + [ + 15.0625 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.59375 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1796875 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.625 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.0625 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.59375 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.8125 + ] + ], + [ + [ + 15.0625 + ] + ], + [ + [ + 3.453125 + ] + ] + ], + "outputs": [ + [ + [ + 2.828125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.16.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 14.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.16.feed_forward.down_proj": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ], + "outputs": [ + [ + [ + 6.65625 + ] + ], + [ + [ + 27.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.423828125 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.125 + ] + ] + ], + "outputs": [ + [ + [ + 19.5 + ] + ], + [ + [ + 27.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.03125 + ] + ], + [ + [ + 16.75 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.125 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.03125 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.40625 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.75 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.125 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.9375 + ] + ], + [ + [ + 15.75 + ] + ], + [ + [ + 5.125 + ] + ] + ], + "outputs": [ + [ + [ + 4.125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.17.feed_forward.router": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 9.0625 + ] + ], + [ + [ + 55.0 + ] + ], + [ + [ + 76.5 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 47.25 + ] + ], + [ + [ + 63.5 + ] + ], + [ + [ + 56.5 + ] + ], + [ + [ + 66.5 + ] + ], + [ + [ + 52.75 + ] + ], + [ + [ + 52.25 + ] + ], + [ + [ + 51.0 + ] + ], + [ + [ + 60.25 + ] + ], + [ + [ + 64.5 + ] + ], + [ + [ + 76.5 + ] + ], + [ + [ + 72.0 + ] + ], + [ + [ + 76.5 + ] + ], + [ + [ + 70.5 + ] + ] + ] + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3046875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.322265625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3046875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10693359375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 21.5 + ] + ] + ], + "outputs": [ + [ + [ + 5.28125 + ] + ], + [ + [ + 1.0035671368176115e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.1875 + ] + ] + ], + "outputs": [ + [ + [ + 6.625 + ] + ], + [ + [ + 12.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.97265625 + ] + ], + [ + [ + 18.75 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.1875 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.97265625 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.1875 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.75 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.1875 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.3125 + ] + ], + [ + [ + 18.75 + ] + ], + [ + [ + 7.71875 + ] + ] + ], + "outputs": [ + [ + [ + 6.78125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.18.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 20.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.59375 + ] + ] + } + }, + "language_model.model.layers.18.feed_forward.down_proj": { + "inputs": [ + [ + [ + 32.5 + ] + ] + ], + "outputs": [ + [ + [ + 8.1875 + ] + ], + [ + [ + 354.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.64453125 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 44.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.8125 + ] + ] + ], + "outputs": [ + [ + [ + 31.5 + ] + ], + [ + [ + 37.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.283203125 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0390625 + ] + ], + [ + [ + 16.375 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.1875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0390625 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.03125 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.375 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.1875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.4375 + ] + ], + [ + [ + 16.375 + ] + ], + [ + [ + 7.125 + ] + ] + ], + "outputs": [ + [ + [ + 5.8125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.19.feed_forward.router": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "outputs": [ + [ + [ + 10.75 + ] + ], + [ + [ + 85.0 + ] + ], + [ + [ + 32.25 + ] + ], + [ + [ + 72.5 + ] + ], + [ + [ + 64.0 + ] + ], + [ + [ + 87.0 + ] + ], + [ + [ + 70.5 + ] + ], + [ + [ + 76.5 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 109.5 + ] + ], + [ + [ + 23.875 + ] + ], + [ + [ + 44.25 + ] + ], + [ + [ + 74.5 + ] + ], + [ + [ + 67.0 + ] + ], + [ + [ + 83.5 + ] + ], + [ + [ + 46.0 + ] + ], + [ + [ + 64.5 + ] + ] + ] + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.439453125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33203125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2353515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.52734375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.466796875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.490234375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.111328125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.423828125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 20.625 + ] + ] + ], + "outputs": [ + [ + [ + 14.8125 + ] + ], + [ + [ + 1.8609191940988822e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.77734375 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 53.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.o_proj": { + "inputs": [ + [ + [ + 9.25 + ] + ] + ], + "outputs": [ + [ + [ + 12.8125 + ] + ], + [ + [ + 22.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.125 + ] + ], + [ + [ + 17.0 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.25 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.125 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.1875 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.0 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.25 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.1875 + ] + ], + [ + [ + 17.0 + ] + ], + [ + [ + 10.25 + ] + ] + ], + "outputs": [ + [ + [ + 9.25 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.20.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 42.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.20.feed_forward.down_proj": { + "inputs": [ + [ + [ + 16.875 + ] + ] + ], + "outputs": [ + [ + [ + 10.5 + ] + ], + [ + [ + 41.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.95703125 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 58.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.3125 + ] + ] + ], + "outputs": [ + [ + [ + 26.0 + ] + ], + [ + [ + 35.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.388671875 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.34375 + ] + ], + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.5 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.34375 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.203125 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.5 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.875 + ] + ], + [ + [ + 17.5 + ] + ], + [ + [ + 8.5 + ] + ] + ], + "outputs": [ + [ + [ + 4.3125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.21.feed_forward.router": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.45703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "outputs": [ + [ + [ + 8.125 + ] + ], + [ + [ + 43.0 + ] + ], + [ + [ + 75.5 + ] + ], + [ + [ + 91.5 + ] + ], + [ + [ + 83.5 + ] + ], + [ + [ + 46.5 + ] + ], + [ + [ + 75.5 + ] + ], + [ + [ + 62.5 + ] + ], + [ + [ + 81.0 + ] + ], + [ + [ + 65.5 + ] + ], + [ + [ + 46.0 + ] + ], + [ + [ + 72.5 + ] + ], + [ + [ + 75.0 + ] + ], + [ + [ + 56.5 + ] + ], + [ + [ + 46.25 + ] + ], + [ + [ + 25.75 + ] + ], + [ + [ + 37.25 + ] + ] + ] + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2353515625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2490234375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.369140625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 43.75 + ] + ] + ], + "outputs": [ + [ + [ + 44.75 + ] + ], + [ + [ + 1.5086737752158795e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.9609375 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 57.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.34375 + ] + ] + ], + "outputs": [ + [ + [ + 15.75 + ] + ], + [ + [ + 28.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0078125 + ] + ], + [ + [ + 19.5 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.8125 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0078125 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.6875 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.5 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.8125 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.875 + ] + ], + [ + [ + 19.5 + ] + ], + [ + [ + 7.8125 + ] + ] + ], + "outputs": [ + [ + [ + 7.34375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.22.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 46.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.421875 + ] + ] + } + }, + "language_model.model.layers.22.feed_forward.down_proj": { + "inputs": [ + [ + [ + 39.25 + ] + ] + ], + "outputs": [ + [ + [ + 14.875 + ] + ], + [ + [ + 77.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6015625 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 69.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.09375 + ] + ] + ], + "outputs": [ + [ + [ + 70.0 + ] + ], + [ + [ + 79.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0546875 + ] + ], + [ + [ + 15.6875 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.9375 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0546875 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.71875 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.6875 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.9375 + ] + ], + [ + [ + 15.625 + ] + ], + [ + [ + 12.9375 + ] + ] + ], + "outputs": [ + [ + [ + 6.09375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.23.feed_forward.router": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.474609375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 12.375 + ] + ], + [ + [ + 34.25 + ] + ], + [ + [ + 96.5 + ] + ], + [ + [ + 56.75 + ] + ], + [ + [ + 95.5 + ] + ], + [ + [ + 34.5 + ] + ], + [ + [ + 55.5 + ] + ], + [ + [ + 42.25 + ] + ], + [ + [ + 51.0 + ] + ], + [ + [ + 54.5 + ] + ], + [ + [ + 43.75 + ] + ], + [ + [ + 39.75 + ] + ], + [ + [ + 91.5 + ] + ], + [ + [ + 54.25 + ] + ], + [ + [ + 50.5 + ] + ], + [ + [ + 24.375 + ] + ], + [ + [ + 56.5 + ] + ] + ] + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.369140625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37109375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 35.5 + ] + ] + ], + "outputs": [ + [ + [ + 31.375 + ] + ], + [ + [ + 1.8609191940988822e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8671875 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.4375 + ] + ] + ], + "outputs": [ + [ + [ + 48.0 + ] + ], + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.640625 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.265625 + ] + ], + [ + [ + 20.375 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.25 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.265625 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.5625 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.375 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.25 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.1875 + ] + ], + [ + [ + 20.375 + ] + ], + [ + [ + 8.125 + ] + ] + ], + "outputs": [ + [ + [ + 7.4375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.24.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.451171875 + ] + ] + } + }, + "language_model.model.layers.24.feed_forward.down_proj": { + "inputs": [ + [ + [ + 27.0 + ] + ] + ], + "outputs": [ + [ + [ + 21.75 + ] + ], + [ + [ + 100.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.59765625 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.34375 + ] + ] + ], + "outputs": [ + [ + [ + 53.0 + ] + ], + [ + [ + 69.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.482421875 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.140625 + ] + ], + [ + [ + 17.875 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.84375 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.140625 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.34375 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.875 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.84375 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.3125 + ] + ], + [ + [ + 17.75 + ] + ], + [ + [ + 6.84375 + ] + ] + ], + "outputs": [ + [ + [ + 5.09375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.25.feed_forward.router": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.412109375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "outputs": [ + [ + [ + 8.5 + ] + ], + [ + [ + 42.75 + ] + ], + [ + [ + 52.25 + ] + ], + [ + [ + 48.0 + ] + ], + [ + [ + 100.0 + ] + ], + [ + [ + 64.0 + ] + ], + [ + [ + 127.0 + ] + ], + [ + [ + 76.5 + ] + ], + [ + [ + 55.25 + ] + ], + [ + [ + 70.5 + ] + ], + [ + [ + 69.0 + ] + ], + [ + [ + 89.0 + ] + ], + [ + [ + 70.5 + ] + ], + [ + [ + 58.5 + ] + ], + [ + [ + 56.5 + ] + ], + [ + [ + 57.5 + ] + ], + [ + [ + 45.75 + ] + ] + ] + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23828125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.484375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 38.75 + ] + ] + ], + "outputs": [ + [ + [ + 7.9375 + ] + ], + [ + [ + 1.8609191940988822e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.60546875 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 63.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.9375 + ] + ] + ], + "outputs": [ + [ + [ + 30.875 + ] + ], + [ + [ + 48.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.25 + ] + ], + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.0625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.25 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.21875 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.0625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.3125 + ] + ], + [ + [ + 18.625 + ] + ], + [ + [ + 8.0625 + ] + ] + ], + "outputs": [ + [ + [ + 6.9375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.26.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 45.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5703125 + ] + ] + } + }, + "language_model.model.layers.26.feed_forward.down_proj": { + "inputs": [ + [ + [ + 42.0 + ] + ] + ], + "outputs": [ + [ + [ + 22.125 + ] + ], + [ + [ + 113.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5234375 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.625 + ] + ] + ], + "outputs": [ + [ + [ + 60.25 + ] + ], + [ + [ + 71.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1640625 + ] + ], + [ + [ + 16.25 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.0625 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1640625 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.9375 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.25 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.0625 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.9375 + ] + ], + [ + [ + 16.25 + ] + ], + [ + [ + 10.0625 + ] + ] + ], + "outputs": [ + [ + [ + 6.625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.27.feed_forward.router": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "outputs": [ + [ + [ + 11.625 + ] + ], + [ + [ + 69.5 + ] + ], + [ + [ + 39.75 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 81.0 + ] + ], + [ + [ + 95.0 + ] + ], + [ + [ + 34.5 + ] + ], + [ + [ + 31.75 + ] + ], + [ + [ + 94.0 + ] + ], + [ + [ + 59.0 + ] + ], + [ + [ + 48.75 + ] + ], + [ + [ + 74.5 + ] + ], + [ + [ + 111.5 + ] + ], + [ + [ + 81.0 + ] + ], + [ + [ + 58.75 + ] + ], + [ + [ + 69.0 + ] + ], + [ + [ + 56.5 + ] + ] + ] + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11767578125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.38671875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1083984375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3046875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.240234375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33984375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 169.0 + ] + ] + ], + "outputs": [ + [ + [ + 111.5 + ] + ], + [ + [ + 5.483065482612778e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.66015625 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 78.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.o_proj": { + "inputs": [ + [ + [ + 9.625 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 97.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.72265625 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.5 + ] + ], + [ + [ + 21.375 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.0625 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.5 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.5625 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 21.375 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.0625 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 17.5 + ] + ], + [ + [ + 21.375 + ] + ], + [ + [ + 10.0625 + ] + ] + ], + "outputs": [ + [ + [ + 9.625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.28.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.349609375 + ] + ] + } + }, + "language_model.model.layers.28.feed_forward.down_proj": { + "inputs": [ + [ + [ + 39.5 + ] + ] + ], + "outputs": [ + [ + [ + 140.0 + ] + ], + [ + [ + 149.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.453125 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 67.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.65625 + ] + ] + ], + "outputs": [ + [ + [ + 78.0 + ] + ], + [ + [ + 84.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.390625 + ] + ], + [ + [ + 20.875 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.1875 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.390625 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.78125 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.875 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.1875 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.875 + ] + ], + [ + [ + 20.125 + ] + ], + [ + [ + 7.1875 + ] + ] + ], + "outputs": [ + [ + [ + 5.65625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.29.feed_forward.router": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.333984375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "outputs": [ + [ + [ + 23.625 + ] + ], + [ + [ + 54.75 + ] + ], + [ + [ + 79.5 + ] + ], + [ + [ + 22.25 + ] + ], + [ + [ + 47.0 + ] + ], + [ + [ + 57.25 + ] + ], + [ + [ + 73.0 + ] + ], + [ + [ + 54.5 + ] + ], + [ + [ + 52.5 + ] + ], + [ + [ + 35.0 + ] + ], + [ + [ + 142.0 + ] + ], + [ + [ + 80.0 + ] + ], + [ + [ + 44.75 + ] + ], + [ + [ + 58.75 + ] + ], + [ + [ + 64.0 + ] + ], + [ + [ + 87.0 + ] + ], + [ + [ + 270.0 + ] + ] + ] + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.373046875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.310546875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2490234375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.392578125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4765625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 44.25 + ] + ] + ], + "outputs": [ + [ + [ + 20.875 + ] + ], + [ + [ + 1.8609191940988822e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.85546875 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.21875 + ] + ] + ], + "outputs": [ + [ + [ + 50.25 + ] + ], + [ + [ + 56.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.171875 + ] + ], + [ + [ + 19.0 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.46875 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.171875 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.6875 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.0 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.46875 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.1875 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 7.46875 + ] + ] + ], + "outputs": [ + [ + [ + 6.21875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.30.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 32.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4453125 + ] + ] + } + }, + "language_model.model.layers.30.feed_forward.down_proj": { + "inputs": [ + [ + [ + 56.25 + ] + ] + ], + "outputs": [ + [ + [ + 30.25 + ] + ], + [ + [ + 159.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.58203125 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.09375 + ] + ] + ], + "outputs": [ + [ + [ + 77.0 + ] + ], + [ + [ + 115.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8046875 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.91796875 + ] + ], + [ + [ + 13.125 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.3125 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.91796875 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.15625 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.125 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.3125 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.375 + ] + ], + [ + [ + 13.125 + ] + ], + [ + [ + 7.3125 + ] + ] + ], + "outputs": [ + [ + [ + 6.09375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.31.feed_forward.router": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "outputs": [ + [ + [ + 30.0 + ] + ], + [ + [ + 49.75 + ] + ], + [ + [ + 114.0 + ] + ], + [ + [ + 129.0 + ] + ], + [ + [ + 101.0 + ] + ], + [ + [ + 73.5 + ] + ], + [ + [ + 72.5 + ] + ], + [ + [ + 109.0 + ] + ], + [ + [ + 34.75 + ] + ], + [ + [ + 96.0 + ] + ], + [ + [ + 125.5 + ] + ], + [ + [ + 74.5 + ] + ], + [ + [ + 59.25 + ] + ], + [ + [ + 72.5 + ] + ], + [ + [ + 35.75 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 46.25 + ] + ] + ] + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.330078125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.447265625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.45703125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.46484375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.345703125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.365234375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.357421875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.439453125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.390625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 41.0 + ] + ] + ], + "outputs": [ + [ + [ + 9.6875 + ] + ], + [ + [ + 1.0035671368176115e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.66796875 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 53.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.3125 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 109.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2734375 + ] + ], + [ + [ + 19.0 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.21875 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2734375 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.0625 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.0 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.21875 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.75 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 7.21875 + ] + ] + ], + "outputs": [ + [ + [ + 6.3125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.32.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 32.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.32.feed_forward.down_proj": { + "inputs": [ + [ + [ + 49.75 + ] + ] + ], + "outputs": [ + [ + [ + 29.375 + ] + ], + [ + [ + 75.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.625 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 44.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.9375 + ] + ] + ], + "outputs": [ + [ + [ + 94.0 + ] + ], + [ + [ + 132.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.25 + ] + ], + [ + [ + 23.375 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.4375 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.25 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.84375 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.4375 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.9375 + ] + ], + [ + [ + 23.375 + ] + ], + [ + [ + 9.4375 + ] + ] + ], + "outputs": [ + [ + [ + 7.9375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.33.feed_forward.router": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "outputs": [ + [ + [ + 17.75 + ] + ], + [ + [ + 112.0 + ] + ], + [ + [ + 138.0 + ] + ], + [ + [ + 106.5 + ] + ], + [ + [ + 129.0 + ] + ], + [ + [ + 67.0 + ] + ], + [ + [ + 143.0 + ] + ], + [ + [ + 77.0 + ] + ], + [ + [ + 40.75 + ] + ], + [ + [ + 145.0 + ] + ], + [ + [ + 105.0 + ] + ], + [ + [ + 76.5 + ] + ], + [ + [ + 92.5 + ] + ], + [ + [ + 57.0 + ] + ], + [ + [ + 97.5 + ] + ], + [ + [ + 85.5 + ] + ], + [ + [ + 134.0 + ] + ] + ] + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0908203125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.345703125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.423828125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 81.5 + ] + ] + ], + "outputs": [ + [ + [ + 22.5 + ] + ], + [ + [ + 1.6415965747943711e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.89453125 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 54.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.1875 + ] + ] + ], + "outputs": [ + [ + [ + 55.75 + ] + ], + [ + [ + 98.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.4765625 + ] + ], + [ + [ + 20.0 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.1875 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4765625 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.1875 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.0 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.1875 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 18.5 + ] + ], + [ + [ + 20.0 + ] + ], + [ + [ + 10.1875 + ] + ] + ], + "outputs": [ + [ + [ + 8.125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.34.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.34.feed_forward.down_proj": { + "inputs": [ + [ + [ + 76.0 + ] + ] + ], + "outputs": [ + [ + [ + 36.5 + ] + ], + [ + [ + 162.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.48828125 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 54.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.125 + ] + ] + ], + "outputs": [ + [ + [ + 110.5 + ] + ], + [ + [ + 116.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2734375 + ] + ], + [ + [ + 14.0 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.875 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2734375 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.9375 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.0 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.875 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.4375 + ] + ], + [ + [ + 14.0 + ] + ], + [ + [ + 7.875 + ] + ] + ], + "outputs": [ + [ + [ + 7.125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.35.feed_forward.router": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "outputs": [ + [ + [ + 26.75 + ] + ], + [ + [ + 122.5 + ] + ], + [ + [ + 142.0 + ] + ], + [ + [ + 132.0 + ] + ], + [ + [ + 59.5 + ] + ], + [ + [ + 162.0 + ] + ], + [ + [ + 146.0 + ] + ], + [ + [ + 71.0 + ] + ], + [ + [ + 54.75 + ] + ], + [ + [ + 52.25 + ] + ], + [ + [ + 77.5 + ] + ], + [ + [ + 117.0 + ] + ], + [ + [ + 114.0 + ] + ], + [ + [ + 116.0 + ] + ], + [ + [ + 152.0 + ] + ], + [ + [ + 135.0 + ] + ], + [ + [ + 101.5 + ] + ] + ] + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2490234375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3203125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.349609375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4296875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.431640625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 89.5 + ] + ] + ], + "outputs": [ + [ + [ + 49.5 + ] + ], + [ + [ + 1.8609191940988822e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.796875 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 56.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.65625 + ] + ] + ], + "outputs": [ + [ + [ + 36.75 + ] + ], + [ + [ + 57.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.515625 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2265625 + ] + ], + [ + [ + 19.5 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.5625 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2265625 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.59375 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.5 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.5625 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.25 + ] + ], + [ + [ + 19.5 + ] + ], + [ + [ + 8.1875 + ] + ] + ], + "outputs": [ + [ + [ + 7.65625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.36.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 29.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.36.feed_forward.down_proj": { + "inputs": [ + [ + [ + 79.5 + ] + ] + ], + "outputs": [ + [ + [ + 43.0 + ] + ], + [ + [ + 94.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.439453125 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.625 + ] + ] + ], + "outputs": [ + [ + [ + 97.5 + ] + ], + [ + [ + 150.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.3984375 + ] + ], + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.5625 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.3984375 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.125 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.625 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.5625 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.0625 + ] + ], + [ + [ + 18.625 + ] + ], + [ + [ + 9.5625 + ] + ] + ], + "outputs": [ + [ + [ + 8.625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.37.feed_forward.router": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "outputs": [ + [ + [ + 41.5 + ] + ], + [ + [ + 147.0 + ] + ], + [ + [ + 64.0 + ] + ], + [ + [ + 124.5 + ] + ], + [ + [ + 101.5 + ] + ], + [ + [ + 110.5 + ] + ], + [ + [ + 101.0 + ] + ], + [ + [ + 153.0 + ] + ], + [ + [ + 118.0 + ] + ], + [ + [ + 121.5 + ] + ], + [ + [ + 200.0 + ] + ], + [ + [ + 56.5 + ] + ], + [ + [ + 100.5 + ] + ], + [ + [ + 91.0 + ] + ], + [ + [ + 142.0 + ] + ], + [ + [ + 58.5 + ] + ], + [ + [ + 101.5 + ] + ] + ] + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.287109375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.421875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.357421875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 103.5 + ] + ] + ], + "outputs": [ + [ + [ + 35.25 + ] + ], + [ + [ + 1.6415965747943711e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.75390625 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 47.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.34375 + ] + ] + ], + "outputs": [ + [ + [ + 29.75 + ] + ], + [ + [ + 66.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1044921875 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.4453125 + ] + ], + [ + [ + 16.75 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.28125 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4453125 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.8125 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.75 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.28125 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 18.875 + ] + ], + [ + [ + 16.75 + ] + ], + [ + [ + 7.28125 + ] + ] + ], + "outputs": [ + [ + [ + 6.34375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.38.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 31.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.38.feed_forward.down_proj": { + "inputs": [ + [ + [ + 96.0 + ] + ] + ], + "outputs": [ + [ + [ + 31.125 + ] + ], + [ + [ + 262.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8046875 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 52.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.o_proj": { + "inputs": [ + [ + [ + 9.1875 + ] + ] + ], + "outputs": [ + [ + [ + 126.5 + ] + ], + [ + [ + 154.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.328125 + ] + ], + [ + [ + 15.4375 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.25 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.328125 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.0 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.4375 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.25 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.625 + ] + ], + [ + [ + 15.4375 + ] + ], + [ + [ + 10.25 + ] + ] + ], + "outputs": [ + [ + [ + 9.1875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.39.feed_forward.router": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "outputs": [ + [ + [ + 38.5 + ] + ], + [ + [ + 108.5 + ] + ], + [ + [ + 176.0 + ] + ], + [ + [ + 129.0 + ] + ], + [ + [ + 124.0 + ] + ], + [ + [ + 138.0 + ] + ], + [ + [ + 128.0 + ] + ], + [ + [ + 205.0 + ] + ], + [ + [ + 119.5 + ] + ], + [ + [ + 116.5 + ] + ], + [ + [ + 90.5 + ] + ], + [ + [ + 141.0 + ] + ], + [ + [ + 108.0 + ] + ], + [ + [ + 144.0 + ] + ], + [ + [ + 89.0 + ] + ], + [ + [ + 190.0 + ] + ], + [ + [ + 121.5 + ] + ] + ] + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1005859375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.404296875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.447265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 172.0 + ] + ] + ], + "outputs": [ + [ + [ + 30.125 + ] + ], + [ + [ + 1.8609191940988822e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.419921875 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 46.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.o_proj": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "outputs": [ + [ + [ + 94.0 + ] + ], + [ + [ + 102.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.5390625 + ] + ], + [ + [ + 13.6875 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.6875 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.5390625 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.4375 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.6875 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 14.6875 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 17.125 + ] + ], + [ + [ + 13.5625 + ] + ], + [ + [ + 14.6875 + ] + ] + ], + "outputs": [ + [ + [ + 11.375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.40.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.40.feed_forward.down_proj": { + "inputs": [ + [ + [ + 154.0 + ] + ] + ], + "outputs": [ + [ + [ + 28.5 + ] + ], + [ + [ + 127.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.62109375 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 35.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.28125 + ] + ] + ], + "outputs": [ + [ + [ + 160.0 + ] + ], + [ + [ + 177.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2890625 + ] + ], + [ + [ + 16.625 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.125 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2890625 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.875 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.125 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.875 + ] + ], + [ + [ + 16.625 + ] + ], + [ + [ + 8.125 + ] + ] + ], + "outputs": [ + [ + [ + 7.28125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.41.feed_forward.router": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "outputs": [ + [ + [ + 45.75 + ] + ], + [ + [ + 232.0 + ] + ], + [ + [ + 197.0 + ] + ], + [ + [ + 194.0 + ] + ], + [ + [ + 172.0 + ] + ], + [ + [ + 139.0 + ] + ], + [ + [ + 159.0 + ] + ], + [ + [ + 154.0 + ] + ], + [ + [ + 137.0 + ] + ], + [ + [ + 143.0 + ] + ], + [ + [ + 174.0 + ] + ], + [ + [ + 121.0 + ] + ], + [ + [ + 254.0 + ] + ], + [ + [ + 136.0 + ] + ], + [ + [ + 151.0 + ] + ], + [ + [ + 215.0 + ] + ], + [ + [ + 177.0 + ] + ] + ] + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.51171875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.283203125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.287109375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.361328125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.322265625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.39453125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3203125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.388671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 160.0 + ] + ] + ], + "outputs": [ + [ + [ + 34.5 + ] + ], + [ + [ + 1.0035671368176115e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.734375 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 39.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.3125 + ] + ] + ], + "outputs": [ + [ + [ + 65.0 + ] + ], + [ + [ + 146.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.5546875 + ] + ], + [ + [ + 18.5 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.8125 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.5546875 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.53125 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.8125 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 18.125 + ] + ], + [ + [ + 18.5 + ] + ], + [ + [ + 8.8125 + ] + ] + ], + "outputs": [ + [ + [ + 7.3125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.42.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 43.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.42.feed_forward.down_proj": { + "inputs": [ + [ + [ + 180.0 + ] + ] + ], + "outputs": [ + [ + [ + 70.5 + ] + ], + [ + [ + 218.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.50390625 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 43.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.287109375 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.o_proj": { + "inputs": [ + [ + [ + 14.0625 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 232.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.046875 + ] + ], + [ + [ + 19.875 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.875 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.046875 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.5 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.875 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 14.875 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 20.0 + ] + ], + [ + [ + 19.875 + ] + ], + [ + [ + 14.875 + ] + ] + ], + "outputs": [ + [ + [ + 14.0625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.43.feed_forward.router": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "outputs": [ + [ + [ + 45.75 + ] + ], + [ + [ + 199.0 + ] + ], + [ + [ + 163.0 + ] + ], + [ + [ + 243.0 + ] + ], + [ + [ + 128.0 + ] + ], + [ + [ + 74.5 + ] + ], + [ + [ + 141.0 + ] + ], + [ + [ + 145.0 + ] + ], + [ + [ + 154.0 + ] + ], + [ + [ + 184.0 + ] + ], + [ + [ + 213.0 + ] + ], + [ + [ + 181.0 + ] + ], + [ + [ + 82.5 + ] + ], + [ + [ + 199.0 + ] + ], + [ + [ + 274.0 + ] + ], + [ + [ + 128.0 + ] + ], + [ + [ + 162.0 + ] + ] + ] + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.345703125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3828125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23828125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 282.0 + ] + ] + ], + "outputs": [ + [ + [ + 37.0 + ] + ], + [ + [ + 1.0035671368176115e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5625 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 29.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.o_proj": { + "inputs": [ + [ + [ + 12.1875 + ] + ] + ], + "outputs": [ + [ + [ + 81.0 + ] + ], + [ + [ + 170.0 + ] + ] + ], + "params": { + "weight": [ + [ + 1.2890625 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.78125 + ] + ], + [ + [ + 10.6875 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.375 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.78125 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.625 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 10.6875 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.375 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 22.5 + ] + ], + [ + [ + 10.6875 + ] + ], + [ + [ + 13.375 + ] + ] + ], + "outputs": [ + [ + [ + 12.1875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.44.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 42.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.44.feed_forward.down_proj": { + "inputs": [ + [ + [ + 282.0 + ] + ] + ], + "outputs": [ + [ + [ + 75.5 + ] + ], + [ + [ + 200.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.43359375 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 28.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.o_proj": { + "inputs": [ + [ + [ + 11.8125 + ] + ] + ], + "outputs": [ + [ + [ + 170.0 + ] + ], + [ + [ + 258.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28515625 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.3125 + ] + ], + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.875 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.3125 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.25 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 12.875 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.3125 + ] + ], + [ + [ + 17.5 + ] + ], + [ + [ + 12.875 + ] + ] + ], + "outputs": [ + [ + [ + 11.8125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.45.feed_forward.router": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "outputs": [ + [ + [ + 29.875 + ] + ], + [ + [ + 52.75 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 42.25 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 103.0 + ] + ], + [ + [ + 222.0 + ] + ], + [ + [ + 140.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 46.0 + ] + ], + [ + [ + 56.25 + ] + ], + [ + [ + 0.0 + ] + ] + ] + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.419921875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33984375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10986328125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.333984375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.310546875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 282.0 + ] + ] + ], + "outputs": [ + [ + [ + 148.0 + ] + ], + [ + [ + 1.8609191940988822e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.70703125 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 27.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.o_proj": { + "inputs": [ + [ + [ + 32.5 + ] + ] + ], + "outputs": [ + [ + [ + 126.0 + ] + ], + [ + [ + 221.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.65234375 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.765625 + ] + ], + [ + [ + 26.375 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 32.5 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.765625 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 31.875 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 26.375 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 32.5 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 24.375 + ] + ], + [ + [ + 26.375 + ] + ], + [ + [ + 32.5 + ] + ] + ], + "outputs": [ + [ + [ + 32.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.46.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 40.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37109375 + ] + ] + } + }, + "language_model.model.layers.46.feed_forward.down_proj": { + "inputs": [ + [ + [ + 1448.0 + ] + ] + ], + "outputs": [ + [ + [ + 1680.0 + ] + ], + [ + [ + 2400.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.498046875 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.o_proj": { + "inputs": [ + [ + [ + 10.6875 + ] + ] + ], + "outputs": [ + [ + [ + 96.5 + ] + ], + [ + [ + 125.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28515625 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.5390625 + ] + ], + [ + [ + 15.125 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 15.4375 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.5390625 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.8125 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.125 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 15.4375 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.1875 + ] + ], + [ + [ + 15.125 + ] + ], + [ + [ + 15.4375 + ] + ] + ], + "outputs": [ + [ + [ + 10.6875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.47.feed_forward.router": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "outputs": [ + [ + [ + 1328.0 + ] + ], + [ + [ + 836.0 + ] + ], + [ + [ + 404.0 + ] + ], + [ + [ + 2752.0 + ] + ], + [ + [ + 253.0 + ] + ], + [ + [ + 89.5 + ] + ], + [ + [ + 206.0 + ] + ], + [ + [ + 294.0 + ] + ], + [ + [ + 55.5 + ] + ], + [ + [ + 162.0 + ] + ], + [ + [ + 458.0 + ] + ], + [ + [ + 229.0 + ] + ], + [ + [ + 392.0 + ] + ], + [ + [ + 548.0 + ] + ], + [ + [ + 70.5 + ] + ], + [ + [ + 102.5 + ] + ], + [ + [ + 204.0 + ] + ] + ] + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1044921875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1005859375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.58203125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.89453125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.83984375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.828125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.57421875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.52734375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.76953125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.345703125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.765625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.78125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.69921875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.765625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7421875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.357421875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.486328125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.384765625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 1968.0 + ] + ] + ], + "outputs": [ + [ + [ + 1488.0 + ] + ], + [ + [ + 1.8609191940988822e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.408203125 + ] + ] + } + }, + "language_model.lm_head": { + "inputs": [ + [ + [ + 54.25 + ] + ] + ], + "params": { + "weight": [ + [ + 1.0234375 + ] + ] + } + } + } +} \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_5_8.npz b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_5_8.npz new file mode 100644 index 000000000000..b1efa4a1b776 Binary files /dev/null and b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_5_8.npz differ diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_5_8_mod_list.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_5_8_mod_list.json new file mode 100644 index 000000000000..449fce78b77d --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_5_8_mod_list.json @@ -0,0 +1,1559 @@ +[ + "vision_model.patch_embedding.linear", + "vision_model.model.layers.0.self_attn.qkv_proj", + "vision_model.model.layers.0.self_attn.o_proj", + "vision_model.model.layers.0.mlp.fc1", + "vision_model.model.layers.0.mlp.fc2", + "vision_model.model.layers.1.self_attn.qkv_proj", + "vision_model.model.layers.1.self_attn.o_proj", + "vision_model.model.layers.1.mlp.fc1", + "vision_model.model.layers.1.mlp.fc2", + "vision_model.model.layers.2.self_attn.qkv_proj", + "vision_model.model.layers.2.self_attn.o_proj", + "vision_model.model.layers.2.mlp.fc1", + "vision_model.model.layers.2.mlp.fc2", + "vision_model.model.layers.3.self_attn.qkv_proj", + "vision_model.model.layers.3.self_attn.o_proj", + "vision_model.model.layers.3.mlp.fc1", + "vision_model.model.layers.3.mlp.fc2", + "vision_model.model.layers.4.self_attn.qkv_proj", + "vision_model.model.layers.4.self_attn.o_proj", + "vision_model.model.layers.4.mlp.fc1", + "vision_model.model.layers.4.mlp.fc2", + "vision_model.model.layers.5.self_attn.qkv_proj", + "vision_model.model.layers.5.self_attn.o_proj", + "vision_model.model.layers.5.mlp.fc1", + "vision_model.model.layers.5.mlp.fc2", + "vision_model.model.layers.6.self_attn.qkv_proj", + "vision_model.model.layers.6.self_attn.o_proj", + "vision_model.model.layers.6.mlp.fc1", + "vision_model.model.layers.6.mlp.fc2", + "vision_model.model.layers.7.self_attn.qkv_proj", + "vision_model.model.layers.7.self_attn.o_proj", + "vision_model.model.layers.7.mlp.fc1", + "vision_model.model.layers.7.mlp.fc2", + "vision_model.model.layers.8.self_attn.qkv_proj", + "vision_model.model.layers.8.self_attn.o_proj", + "vision_model.model.layers.8.mlp.fc1", + "vision_model.model.layers.8.mlp.fc2", + "vision_model.model.layers.9.self_attn.qkv_proj", + "vision_model.model.layers.9.self_attn.o_proj", + "vision_model.model.layers.9.mlp.fc1", + "vision_model.model.layers.9.mlp.fc2", + "vision_model.model.layers.10.self_attn.qkv_proj", + "vision_model.model.layers.10.self_attn.o_proj", + "vision_model.model.layers.10.mlp.fc1", + "vision_model.model.layers.10.mlp.fc2", + "vision_model.model.layers.11.self_attn.qkv_proj", + "vision_model.model.layers.11.self_attn.o_proj", + "vision_model.model.layers.11.mlp.fc1", + "vision_model.model.layers.11.mlp.fc2", + "vision_model.model.layers.12.self_attn.qkv_proj", + "vision_model.model.layers.12.self_attn.o_proj", + "vision_model.model.layers.12.mlp.fc1", + "vision_model.model.layers.12.mlp.fc2", + "vision_model.model.layers.13.self_attn.qkv_proj", + "vision_model.model.layers.13.self_attn.o_proj", + "vision_model.model.layers.13.mlp.fc1", + "vision_model.model.layers.13.mlp.fc2", + "vision_model.model.layers.14.self_attn.qkv_proj", + "vision_model.model.layers.14.self_attn.o_proj", + "vision_model.model.layers.14.mlp.fc1", + "vision_model.model.layers.14.mlp.fc2", + "vision_model.model.layers.15.self_attn.qkv_proj", + "vision_model.model.layers.15.self_attn.o_proj", + "vision_model.model.layers.15.mlp.fc1", + "vision_model.model.layers.15.mlp.fc2", + "vision_model.model.layers.16.self_attn.qkv_proj", + "vision_model.model.layers.16.self_attn.o_proj", + "vision_model.model.layers.16.mlp.fc1", + "vision_model.model.layers.16.mlp.fc2", + "vision_model.model.layers.17.self_attn.qkv_proj", + "vision_model.model.layers.17.self_attn.o_proj", + "vision_model.model.layers.17.mlp.fc1", + "vision_model.model.layers.17.mlp.fc2", + "vision_model.model.layers.18.self_attn.qkv_proj", + "vision_model.model.layers.18.self_attn.o_proj", + "vision_model.model.layers.18.mlp.fc1", + "vision_model.model.layers.18.mlp.fc2", + "vision_model.model.layers.19.self_attn.qkv_proj", + "vision_model.model.layers.19.self_attn.o_proj", + "vision_model.model.layers.19.mlp.fc1", + "vision_model.model.layers.19.mlp.fc2", + "vision_model.model.layers.20.self_attn.qkv_proj", + "vision_model.model.layers.20.self_attn.o_proj", + "vision_model.model.layers.20.mlp.fc1", + "vision_model.model.layers.20.mlp.fc2", + "vision_model.model.layers.21.self_attn.qkv_proj", + "vision_model.model.layers.21.self_attn.o_proj", + "vision_model.model.layers.21.mlp.fc1", + "vision_model.model.layers.21.mlp.fc2", + "vision_model.model.layers.22.self_attn.qkv_proj", + "vision_model.model.layers.22.self_attn.o_proj", + "vision_model.model.layers.22.mlp.fc1", + "vision_model.model.layers.22.mlp.fc2", + "vision_model.model.layers.23.self_attn.qkv_proj", + "vision_model.model.layers.23.self_attn.o_proj", + "vision_model.model.layers.23.mlp.fc1", + "vision_model.model.layers.23.mlp.fc2", + "vision_model.model.layers.24.self_attn.qkv_proj", + "vision_model.model.layers.24.self_attn.o_proj", + "vision_model.model.layers.24.mlp.fc1", + "vision_model.model.layers.24.mlp.fc2", + "vision_model.model.layers.25.self_attn.qkv_proj", + "vision_model.model.layers.25.self_attn.o_proj", + "vision_model.model.layers.25.mlp.fc1", + "vision_model.model.layers.25.mlp.fc2", + "vision_model.model.layers.26.self_attn.qkv_proj", + "vision_model.model.layers.26.self_attn.o_proj", + "vision_model.model.layers.26.mlp.fc1", + "vision_model.model.layers.26.mlp.fc2", + "vision_model.model.layers.27.self_attn.qkv_proj", + "vision_model.model.layers.27.self_attn.o_proj", + "vision_model.model.layers.27.mlp.fc1", + "vision_model.model.layers.27.mlp.fc2", + "vision_model.model.layers.28.self_attn.qkv_proj", + "vision_model.model.layers.28.self_attn.o_proj", + "vision_model.model.layers.28.mlp.fc1", + "vision_model.model.layers.28.mlp.fc2", + "vision_model.model.layers.29.self_attn.qkv_proj", + "vision_model.model.layers.29.self_attn.o_proj", + "vision_model.model.layers.29.mlp.fc1", + "vision_model.model.layers.29.mlp.fc2", + "vision_model.model.layers.30.self_attn.qkv_proj", + "vision_model.model.layers.30.self_attn.o_proj", + "vision_model.model.layers.30.mlp.fc1", + "vision_model.model.layers.30.mlp.fc2", + "vision_model.model.layers.31.self_attn.qkv_proj", + "vision_model.model.layers.31.self_attn.o_proj", + "vision_model.model.layers.31.mlp.fc1", + "vision_model.model.layers.31.mlp.fc2", + "vision_model.model.layers.32.self_attn.qkv_proj", + "vision_model.model.layers.32.self_attn.o_proj", + "vision_model.model.layers.32.mlp.fc1", + "vision_model.model.layers.32.mlp.fc2", + "vision_model.model.layers.33.self_attn.qkv_proj", + "vision_model.model.layers.33.self_attn.o_proj", + "vision_model.model.layers.33.mlp.fc1", + "vision_model.model.layers.33.mlp.fc2", + "vision_model.vision_adapter.mlp.fc1", + "vision_model.vision_adapter.mlp.fc2", + "multi_modal_projector.linear_1", + "language_model.model.layers.0.self_attn.qkv_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.0.self_attn.attn.impl.softmax", + "language_model.model.layers.0.self_attn.attn.impl.matmul_av", + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.0.self_attn.attn.impl.k_cache", + "language_model.model.layers.0.self_attn.attn.impl.v_cache", + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.0.feed_forward.gate_up_proj", + "language_model.model.layers.0.feed_forward.down_proj", + "language_model.model.layers.1.self_attn.qkv_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.1.self_attn.attn.impl.softmax", + "language_model.model.layers.1.self_attn.attn.impl.matmul_av", + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.1.self_attn.attn.impl.k_cache", + "language_model.model.layers.1.self_attn.attn.impl.v_cache", + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.1.feed_forward.router", + "language_model.model.layers.1.feed_forward.experts", + "language_model.model.layers.1.feed_forward.experts.moe_op", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.1.feed_forward.shared_expert.down_proj", + "language_model.model.layers.2.self_attn.qkv_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.2.self_attn.attn.impl.softmax", + "language_model.model.layers.2.self_attn.attn.impl.matmul_av", + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.2.self_attn.attn.impl.k_cache", + "language_model.model.layers.2.self_attn.attn.impl.v_cache", + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.2.feed_forward.gate_up_proj", + "language_model.model.layers.2.feed_forward.down_proj", + "language_model.model.layers.3.self_attn.qkv_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.3.self_attn.attn.impl.softmax", + "language_model.model.layers.3.self_attn.attn.impl.matmul_av", + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.3.self_attn.attn.impl.k_cache", + "language_model.model.layers.3.self_attn.attn.impl.v_cache", + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.3.feed_forward.router", + "language_model.model.layers.3.feed_forward.experts", + "language_model.model.layers.3.feed_forward.experts.moe_op", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.3.feed_forward.shared_expert.down_proj", + "language_model.model.layers.4.self_attn.qkv_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.4.self_attn.attn.impl.softmax", + "language_model.model.layers.4.self_attn.attn.impl.matmul_av", + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.4.self_attn.attn.impl.k_cache", + "language_model.model.layers.4.self_attn.attn.impl.v_cache", + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.4.feed_forward.gate_up_proj", + "language_model.model.layers.4.feed_forward.down_proj", + "language_model.model.layers.5.self_attn.qkv_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.5.self_attn.attn.impl.softmax", + "language_model.model.layers.5.self_attn.attn.impl.matmul_av", + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.5.self_attn.attn.impl.k_cache", + "language_model.model.layers.5.self_attn.attn.impl.v_cache", + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.5.feed_forward.router", + "language_model.model.layers.5.feed_forward.experts", + "language_model.model.layers.5.feed_forward.experts.moe_op", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.5.feed_forward.shared_expert.down_proj", + "language_model.model.layers.6.self_attn.qkv_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.6.self_attn.attn.impl.softmax", + "language_model.model.layers.6.self_attn.attn.impl.matmul_av", + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.6.self_attn.attn.impl.k_cache", + "language_model.model.layers.6.self_attn.attn.impl.v_cache", + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.6.feed_forward.gate_up_proj", + "language_model.model.layers.6.feed_forward.down_proj", + "language_model.model.layers.7.self_attn.qkv_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.7.self_attn.attn.impl.softmax", + "language_model.model.layers.7.self_attn.attn.impl.matmul_av", + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.7.self_attn.attn.impl.k_cache", + "language_model.model.layers.7.self_attn.attn.impl.v_cache", + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.7.feed_forward.router", + "language_model.model.layers.7.feed_forward.experts", + "language_model.model.layers.7.feed_forward.experts.moe_op", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.7.feed_forward.shared_expert.down_proj", + "language_model.model.layers.8.self_attn.qkv_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.8.self_attn.attn.impl.softmax", + "language_model.model.layers.8.self_attn.attn.impl.matmul_av", + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.8.self_attn.attn.impl.k_cache", + "language_model.model.layers.8.self_attn.attn.impl.v_cache", + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.8.feed_forward.gate_up_proj", + "language_model.model.layers.8.feed_forward.down_proj", + "language_model.model.layers.9.self_attn.qkv_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.9.self_attn.attn.impl.softmax", + "language_model.model.layers.9.self_attn.attn.impl.matmul_av", + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.9.self_attn.attn.impl.k_cache", + "language_model.model.layers.9.self_attn.attn.impl.v_cache", + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.9.feed_forward.router", + "language_model.model.layers.9.feed_forward.experts", + "language_model.model.layers.9.feed_forward.experts.moe_op", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.9.feed_forward.shared_expert.down_proj", + "language_model.model.layers.10.self_attn.qkv_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.10.self_attn.attn.impl.softmax", + "language_model.model.layers.10.self_attn.attn.impl.matmul_av", + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.10.self_attn.attn.impl.k_cache", + "language_model.model.layers.10.self_attn.attn.impl.v_cache", + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.10.feed_forward.gate_up_proj", + "language_model.model.layers.10.feed_forward.down_proj", + "language_model.model.layers.11.self_attn.qkv_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.11.self_attn.attn.impl.softmax", + "language_model.model.layers.11.self_attn.attn.impl.matmul_av", + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.11.self_attn.attn.impl.k_cache", + "language_model.model.layers.11.self_attn.attn.impl.v_cache", + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.11.feed_forward.router", + "language_model.model.layers.11.feed_forward.experts", + "language_model.model.layers.11.feed_forward.experts.moe_op", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.11.feed_forward.shared_expert.down_proj", + "language_model.model.layers.12.self_attn.qkv_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.12.self_attn.attn.impl.softmax", + "language_model.model.layers.12.self_attn.attn.impl.matmul_av", + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.12.self_attn.attn.impl.k_cache", + "language_model.model.layers.12.self_attn.attn.impl.v_cache", + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.12.feed_forward.gate_up_proj", + "language_model.model.layers.12.feed_forward.down_proj", + "language_model.model.layers.13.self_attn.qkv_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.13.self_attn.attn.impl.softmax", + "language_model.model.layers.13.self_attn.attn.impl.matmul_av", + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.13.self_attn.attn.impl.k_cache", + "language_model.model.layers.13.self_attn.attn.impl.v_cache", + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.13.feed_forward.router", + "language_model.model.layers.13.feed_forward.experts", + "language_model.model.layers.13.feed_forward.experts.moe_op", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.13.feed_forward.shared_expert.down_proj", + "language_model.model.layers.14.self_attn.qkv_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.14.self_attn.attn.impl.softmax", + "language_model.model.layers.14.self_attn.attn.impl.matmul_av", + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.14.self_attn.attn.impl.k_cache", + "language_model.model.layers.14.self_attn.attn.impl.v_cache", + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.14.feed_forward.gate_up_proj", + "language_model.model.layers.14.feed_forward.down_proj", + "language_model.model.layers.15.self_attn.qkv_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.15.self_attn.attn.impl.softmax", + "language_model.model.layers.15.self_attn.attn.impl.matmul_av", + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.15.self_attn.attn.impl.k_cache", + "language_model.model.layers.15.self_attn.attn.impl.v_cache", + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.15.feed_forward.router", + "language_model.model.layers.15.feed_forward.experts", + "language_model.model.layers.15.feed_forward.experts.moe_op", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.15.feed_forward.shared_expert.down_proj", + "language_model.model.layers.16.self_attn.qkv_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.16.self_attn.attn.impl.softmax", + "language_model.model.layers.16.self_attn.attn.impl.matmul_av", + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.16.self_attn.attn.impl.k_cache", + "language_model.model.layers.16.self_attn.attn.impl.v_cache", + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.16.feed_forward.gate_up_proj", + "language_model.model.layers.16.feed_forward.down_proj", + "language_model.model.layers.17.self_attn.qkv_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.17.self_attn.attn.impl.softmax", + "language_model.model.layers.17.self_attn.attn.impl.matmul_av", + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.17.self_attn.attn.impl.k_cache", + "language_model.model.layers.17.self_attn.attn.impl.v_cache", + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.17.feed_forward.router", + "language_model.model.layers.17.feed_forward.experts", + "language_model.model.layers.17.feed_forward.experts.moe_op", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.17.feed_forward.shared_expert.down_proj", + "language_model.model.layers.18.self_attn.qkv_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.18.self_attn.attn.impl.softmax", + "language_model.model.layers.18.self_attn.attn.impl.matmul_av", + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.18.self_attn.attn.impl.k_cache", + "language_model.model.layers.18.self_attn.attn.impl.v_cache", + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.18.feed_forward.gate_up_proj", + "language_model.model.layers.18.feed_forward.down_proj", + "language_model.model.layers.19.self_attn.qkv_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.19.self_attn.attn.impl.softmax", + "language_model.model.layers.19.self_attn.attn.impl.matmul_av", + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.19.self_attn.attn.impl.k_cache", + "language_model.model.layers.19.self_attn.attn.impl.v_cache", + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.19.feed_forward.router", + "language_model.model.layers.19.feed_forward.experts", + "language_model.model.layers.19.feed_forward.experts.moe_op", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.19.feed_forward.shared_expert.down_proj", + "language_model.model.layers.20.self_attn.qkv_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.20.self_attn.attn.impl.softmax", + "language_model.model.layers.20.self_attn.attn.impl.matmul_av", + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.20.self_attn.attn.impl.k_cache", + "language_model.model.layers.20.self_attn.attn.impl.v_cache", + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.20.feed_forward.gate_up_proj", + "language_model.model.layers.20.feed_forward.down_proj", + "language_model.model.layers.21.self_attn.qkv_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.21.self_attn.attn.impl.softmax", + "language_model.model.layers.21.self_attn.attn.impl.matmul_av", + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.21.self_attn.attn.impl.k_cache", + "language_model.model.layers.21.self_attn.attn.impl.v_cache", + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.21.feed_forward.router", + "language_model.model.layers.21.feed_forward.experts", + "language_model.model.layers.21.feed_forward.experts.moe_op", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.21.feed_forward.shared_expert.down_proj", + "language_model.model.layers.22.self_attn.qkv_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.22.self_attn.attn.impl.softmax", + "language_model.model.layers.22.self_attn.attn.impl.matmul_av", + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.22.self_attn.attn.impl.k_cache", + "language_model.model.layers.22.self_attn.attn.impl.v_cache", + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.22.feed_forward.gate_up_proj", + "language_model.model.layers.22.feed_forward.down_proj", + "language_model.model.layers.23.self_attn.qkv_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.23.self_attn.attn.impl.softmax", + "language_model.model.layers.23.self_attn.attn.impl.matmul_av", + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.23.self_attn.attn.impl.k_cache", + "language_model.model.layers.23.self_attn.attn.impl.v_cache", + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.23.feed_forward.router", + "language_model.model.layers.23.feed_forward.experts", + "language_model.model.layers.23.feed_forward.experts.moe_op", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.23.feed_forward.shared_expert.down_proj", + "language_model.model.layers.24.self_attn.qkv_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.24.self_attn.attn.impl.softmax", + "language_model.model.layers.24.self_attn.attn.impl.matmul_av", + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.24.self_attn.attn.impl.k_cache", + "language_model.model.layers.24.self_attn.attn.impl.v_cache", + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.24.feed_forward.gate_up_proj", + "language_model.model.layers.24.feed_forward.down_proj", + "language_model.model.layers.25.self_attn.qkv_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.25.self_attn.attn.impl.softmax", + "language_model.model.layers.25.self_attn.attn.impl.matmul_av", + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.25.self_attn.attn.impl.k_cache", + "language_model.model.layers.25.self_attn.attn.impl.v_cache", + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.25.feed_forward.router", + "language_model.model.layers.25.feed_forward.experts", + "language_model.model.layers.25.feed_forward.experts.moe_op", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.25.feed_forward.shared_expert.down_proj", + "language_model.model.layers.26.self_attn.qkv_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.26.self_attn.attn.impl.softmax", + "language_model.model.layers.26.self_attn.attn.impl.matmul_av", + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.26.self_attn.attn.impl.k_cache", + "language_model.model.layers.26.self_attn.attn.impl.v_cache", + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.26.feed_forward.gate_up_proj", + "language_model.model.layers.26.feed_forward.down_proj", + "language_model.model.layers.27.self_attn.qkv_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.27.self_attn.attn.impl.softmax", + "language_model.model.layers.27.self_attn.attn.impl.matmul_av", + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.27.self_attn.attn.impl.k_cache", + "language_model.model.layers.27.self_attn.attn.impl.v_cache", + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.27.feed_forward.router", + "language_model.model.layers.27.feed_forward.experts", + "language_model.model.layers.27.feed_forward.experts.moe_op", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.27.feed_forward.shared_expert.down_proj", + "language_model.model.layers.28.self_attn.qkv_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.28.self_attn.attn.impl.softmax", + "language_model.model.layers.28.self_attn.attn.impl.matmul_av", + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.28.self_attn.attn.impl.k_cache", + "language_model.model.layers.28.self_attn.attn.impl.v_cache", + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.28.feed_forward.gate_up_proj", + "language_model.model.layers.28.feed_forward.down_proj", + "language_model.model.layers.29.self_attn.qkv_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.29.self_attn.attn.impl.softmax", + "language_model.model.layers.29.self_attn.attn.impl.matmul_av", + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.29.self_attn.attn.impl.k_cache", + "language_model.model.layers.29.self_attn.attn.impl.v_cache", + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.29.feed_forward.router", + "language_model.model.layers.29.feed_forward.experts", + "language_model.model.layers.29.feed_forward.experts.moe_op", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.29.feed_forward.shared_expert.down_proj", + "language_model.model.layers.30.self_attn.qkv_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.30.self_attn.attn.impl.softmax", + "language_model.model.layers.30.self_attn.attn.impl.matmul_av", + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.30.self_attn.attn.impl.k_cache", + "language_model.model.layers.30.self_attn.attn.impl.v_cache", + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.30.feed_forward.gate_up_proj", + "language_model.model.layers.30.feed_forward.down_proj", + "language_model.model.layers.31.self_attn.qkv_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.31.self_attn.attn.impl.softmax", + "language_model.model.layers.31.self_attn.attn.impl.matmul_av", + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.31.self_attn.attn.impl.k_cache", + "language_model.model.layers.31.self_attn.attn.impl.v_cache", + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.31.feed_forward.router", + "language_model.model.layers.31.feed_forward.experts", + "language_model.model.layers.31.feed_forward.experts.moe_op", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.31.feed_forward.shared_expert.down_proj", + "language_model.model.layers.32.self_attn.qkv_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.32.self_attn.attn.impl.softmax", + "language_model.model.layers.32.self_attn.attn.impl.matmul_av", + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.32.self_attn.attn.impl.k_cache", + "language_model.model.layers.32.self_attn.attn.impl.v_cache", + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.32.feed_forward.gate_up_proj", + "language_model.model.layers.32.feed_forward.down_proj", + "language_model.model.layers.33.self_attn.qkv_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.33.self_attn.attn.impl.softmax", + "language_model.model.layers.33.self_attn.attn.impl.matmul_av", + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.33.self_attn.attn.impl.k_cache", + "language_model.model.layers.33.self_attn.attn.impl.v_cache", + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.33.feed_forward.router", + "language_model.model.layers.33.feed_forward.experts", + "language_model.model.layers.33.feed_forward.experts.moe_op", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.33.feed_forward.shared_expert.down_proj", + "language_model.model.layers.34.self_attn.qkv_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.34.self_attn.attn.impl.softmax", + "language_model.model.layers.34.self_attn.attn.impl.matmul_av", + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.34.self_attn.attn.impl.k_cache", + "language_model.model.layers.34.self_attn.attn.impl.v_cache", + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.34.feed_forward.gate_up_proj", + "language_model.model.layers.34.feed_forward.down_proj", + "language_model.model.layers.35.self_attn.qkv_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.35.self_attn.attn.impl.softmax", + "language_model.model.layers.35.self_attn.attn.impl.matmul_av", + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.35.self_attn.attn.impl.k_cache", + "language_model.model.layers.35.self_attn.attn.impl.v_cache", + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.35.feed_forward.router", + "language_model.model.layers.35.feed_forward.experts", + "language_model.model.layers.35.feed_forward.experts.moe_op", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.35.feed_forward.shared_expert.down_proj", + "language_model.model.layers.36.self_attn.qkv_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.36.self_attn.attn.impl.softmax", + "language_model.model.layers.36.self_attn.attn.impl.matmul_av", + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.36.self_attn.attn.impl.k_cache", + "language_model.model.layers.36.self_attn.attn.impl.v_cache", + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.36.feed_forward.gate_up_proj", + "language_model.model.layers.36.feed_forward.down_proj", + "language_model.model.layers.37.self_attn.qkv_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.37.self_attn.attn.impl.softmax", + "language_model.model.layers.37.self_attn.attn.impl.matmul_av", + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.37.self_attn.attn.impl.k_cache", + "language_model.model.layers.37.self_attn.attn.impl.v_cache", + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.37.feed_forward.router", + "language_model.model.layers.37.feed_forward.experts", + "language_model.model.layers.37.feed_forward.experts.moe_op", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.37.feed_forward.shared_expert.down_proj", + "language_model.model.layers.38.self_attn.qkv_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.38.self_attn.attn.impl.softmax", + "language_model.model.layers.38.self_attn.attn.impl.matmul_av", + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.38.self_attn.attn.impl.k_cache", + "language_model.model.layers.38.self_attn.attn.impl.v_cache", + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.38.feed_forward.gate_up_proj", + "language_model.model.layers.38.feed_forward.down_proj", + "language_model.model.layers.39.self_attn.qkv_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.39.self_attn.attn.impl.softmax", + "language_model.model.layers.39.self_attn.attn.impl.matmul_av", + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.39.self_attn.attn.impl.k_cache", + "language_model.model.layers.39.self_attn.attn.impl.v_cache", + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.39.feed_forward.router", + "language_model.model.layers.39.feed_forward.experts", + "language_model.model.layers.39.feed_forward.experts.moe_op", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.39.feed_forward.shared_expert.down_proj", + "language_model.model.layers.40.self_attn.qkv_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.40.self_attn.attn.impl.softmax", + "language_model.model.layers.40.self_attn.attn.impl.matmul_av", + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.40.self_attn.attn.impl.k_cache", + "language_model.model.layers.40.self_attn.attn.impl.v_cache", + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.40.feed_forward.gate_up_proj", + "language_model.model.layers.40.feed_forward.down_proj", + "language_model.model.layers.41.self_attn.qkv_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.41.self_attn.attn.impl.softmax", + "language_model.model.layers.41.self_attn.attn.impl.matmul_av", + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.41.self_attn.attn.impl.k_cache", + "language_model.model.layers.41.self_attn.attn.impl.v_cache", + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.41.feed_forward.router", + "language_model.model.layers.41.feed_forward.experts", + "language_model.model.layers.41.feed_forward.experts.moe_op", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.41.feed_forward.shared_expert.down_proj", + "language_model.model.layers.42.self_attn.qkv_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.42.self_attn.attn.impl.softmax", + "language_model.model.layers.42.self_attn.attn.impl.matmul_av", + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.42.self_attn.attn.impl.k_cache", + "language_model.model.layers.42.self_attn.attn.impl.v_cache", + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.42.feed_forward.gate_up_proj", + "language_model.model.layers.42.feed_forward.down_proj", + "language_model.model.layers.43.self_attn.qkv_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.43.self_attn.attn.impl.softmax", + "language_model.model.layers.43.self_attn.attn.impl.matmul_av", + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.43.self_attn.attn.impl.k_cache", + "language_model.model.layers.43.self_attn.attn.impl.v_cache", + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.43.feed_forward.router", + "language_model.model.layers.43.feed_forward.experts", + "language_model.model.layers.43.feed_forward.experts.moe_op", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.43.feed_forward.shared_expert.down_proj", + "language_model.model.layers.44.self_attn.qkv_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.44.self_attn.attn.impl.softmax", + "language_model.model.layers.44.self_attn.attn.impl.matmul_av", + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.44.self_attn.attn.impl.k_cache", + "language_model.model.layers.44.self_attn.attn.impl.v_cache", + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.44.feed_forward.gate_up_proj", + "language_model.model.layers.44.feed_forward.down_proj", + "language_model.model.layers.45.self_attn.qkv_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.45.self_attn.attn.impl.softmax", + "language_model.model.layers.45.self_attn.attn.impl.matmul_av", + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.45.self_attn.attn.impl.k_cache", + "language_model.model.layers.45.self_attn.attn.impl.v_cache", + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.45.feed_forward.router", + "language_model.model.layers.45.feed_forward.experts", + "language_model.model.layers.45.feed_forward.experts.moe_op", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.45.feed_forward.shared_expert.down_proj", + "language_model.model.layers.46.self_attn.qkv_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.46.self_attn.attn.impl.softmax", + "language_model.model.layers.46.self_attn.attn.impl.matmul_av", + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.46.self_attn.attn.impl.k_cache", + "language_model.model.layers.46.self_attn.attn.impl.v_cache", + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.46.feed_forward.gate_up_proj", + "language_model.model.layers.46.feed_forward.down_proj", + "language_model.model.layers.47.self_attn.qkv_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.47.self_attn.attn.impl.softmax", + "language_model.model.layers.47.self_attn.attn.impl.matmul_av", + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.47.self_attn.attn.impl.k_cache", + "language_model.model.layers.47.self_attn.attn.impl.v_cache", + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.47.feed_forward.router", + "language_model.model.layers.47.feed_forward.experts", + "language_model.model.layers.47.feed_forward.experts.moe_op", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.47.feed_forward.shared_expert.down_proj", + "language_model.lm_head" +] \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_6_8.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_6_8.json new file mode 100644 index 000000000000..33dee06b2507 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_6_8.json @@ -0,0 +1,24263 @@ +{ + "GlobalRank": null, + "LocalRank": 6, + "Mode": "DynamicRange", + "Nodes": { + "language_model.model.layers.0.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 11.1875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.431640625 + ] + ] + } + }, + "language_model.model.layers.0.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.3203125 + ] + ] + ], + "outputs": [ + [ + [ + 0.83203125 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.396484375 + ] + ] + } + }, + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.435546875 + ] + ], + [ + [ + 4.71875 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.328125 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.435546875 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.328125 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 4.71875 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.328125 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 4.9375 + ] + ], + [ + [ + 4.71875 + ] + ], + [ + [ + 1.328125 + ] + ] + ], + "outputs": [ + [ + [ + 1.3046875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.0.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 2.546875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.0.feed_forward.down_proj": { + "inputs": [ + [ + [ + 3.671875 + ] + ] + ], + "outputs": [ + [ + [ + 26.75 + ] + ], + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.66015625 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 10.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.515625 + ] + ] + ], + "outputs": [ + [ + [ + 0.67578125 + ] + ], + [ + [ + 1.3984375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.63671875 + ] + ], + [ + [ + 7.53125 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.80859375 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.63671875 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.48046875 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 7.53125 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.80859375 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.1875 + ] + ], + [ + [ + 7.53125 + ] + ], + [ + [ + 0.80859375 + ] + ] + ], + "outputs": [ + [ + [ + 0.515625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.1.feed_forward.router": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "outputs": [ + [ + [ + 1.6015625 + ] + ], + [ + [ + 17.25 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 24.75 + ] + ], + [ + [ + 24.125 + ] + ], + [ + [ + 11.4375 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 12.5 + ] + ], + [ + [ + 16.75 + ] + ], + [ + [ + 3.859375 + ] + ], + [ + [ + 13.5 + ] + ], + [ + [ + 0.12890625 + ] + ], + [ + [ + 0.515625 + ] + ], + [ + [ + 8.0625 + ] + ], + [ + [ + 14.5 + ] + ], + [ + [ + 14.9375 + ] + ], + [ + [ + 5.78125 + ] + ] + ] + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.244140625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5703125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33203125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 6.0 + ] + ] + ], + "outputs": [ + [ + [ + 0.50390625 + ] + ], + [ + [ + 1.5867659199682433e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 9.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.83203125 + ] + ] + ], + "outputs": [ + [ + [ + 0.7734375 + ] + ], + [ + [ + 1.4921875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.361328125 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.5703125 + ] + ], + [ + [ + 9.5 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.359375 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.5703125 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.87109375 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 9.5 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.359375 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.5625 + ] + ], + [ + [ + 9.5 + ] + ], + [ + [ + 1.359375 + ] + ] + ], + "outputs": [ + [ + [ + 0.83203125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.2.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 6.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.2.feed_forward.down_proj": { + "inputs": [ + [ + [ + 6.5 + ] + ] + ], + "outputs": [ + [ + [ + 4.75 + ] + ], + [ + [ + 25.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.578125 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 7.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.435546875 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.2275390625 + ] + ] + ], + "outputs": [ + [ + [ + 0.77734375 + ] + ], + [ + [ + 2.40625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.271484375 + ] + ], + [ + [ + 3.703125 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.296875 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.271484375 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.185546875 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 3.703125 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.296875 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 2.8125 + ] + ], + [ + [ + 3.703125 + ] + ], + [ + [ + 0.296875 + ] + ] + ], + "outputs": [ + [ + [ + 0.2275390625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.3.feed_forward.router": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.52734375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "outputs": [ + [ + [ + 1.4609375 + ] + ], + [ + [ + 16.25 + ] + ], + [ + [ + 17.625 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 8.375 + ] + ], + [ + [ + 19.5 + ] + ], + [ + [ + 21.5 + ] + ], + [ + [ + 17.875 + ] + ], + [ + [ + 17.5 + ] + ], + [ + [ + 28.5 + ] + ], + [ + [ + 16.0 + ] + ], + [ + [ + 16.0 + ] + ], + [ + [ + 25.0 + ] + ], + [ + [ + 12.8125 + ] + ], + [ + [ + 18.25 + ] + ], + [ + [ + 18.375 + ] + ], + [ + [ + 12.0 + ] + ] + ] + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.111328125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10693359375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 17.375 + ] + ] + ], + "outputs": [ + [ + [ + 14.25 + ] + ], + [ + [ + 4.153837486827862e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.81640625 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 14.3125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.28125 + ] + ] + ], + "outputs": [ + [ + [ + 1.46875 + ] + ], + [ + [ + 2.328125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.796875 + ] + ], + [ + [ + 10.375 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.59375 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.796875 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.125 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 10.375 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.59375 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.5 + ] + ], + [ + [ + 10.375 + ] + ], + [ + [ + 2.59375 + ] + ] + ], + "outputs": [ + [ + [ + 2.28125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.4.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 6.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.4.feed_forward.down_proj": { + "inputs": [ + [ + [ + 6.78125 + ] + ] + ], + "outputs": [ + [ + [ + 2.796875 + ] + ], + [ + [ + 5.1875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.419921875 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 18.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.484375 + ] + ] + ], + "outputs": [ + [ + [ + 1.4921875 + ] + ], + [ + [ + 2.40625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.59375 + ] + ], + [ + [ + 14.25 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.84375 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.59375 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.375 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.25 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.84375 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.25 + ] + ], + [ + [ + 14.25 + ] + ], + [ + [ + 3.84375 + ] + ] + ], + "outputs": [ + [ + [ + 3.484375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.5.feed_forward.router": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 2.3125 + ] + ], + [ + [ + 21.875 + ] + ], + [ + [ + 41.0 + ] + ], + [ + [ + 19.125 + ] + ], + [ + [ + 28.875 + ] + ], + [ + [ + 20.0 + ] + ], + [ + [ + 19.25 + ] + ], + [ + [ + 15.875 + ] + ], + [ + [ + 34.5 + ] + ], + [ + [ + 33.5 + ] + ], + [ + [ + 32.25 + ] + ], + [ + [ + 12.4375 + ] + ], + [ + [ + 26.25 + ] + ], + [ + [ + 25.25 + ] + ], + [ + [ + 15.25 + ] + ], + [ + [ + 16.75 + ] + ], + [ + [ + 19.375 + ] + ] + ] + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.091796875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08251953125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 7.125 + ] + ] + ], + "outputs": [ + [ + [ + 0.4921875 + ] + ], + [ + [ + 1.5867659199682433e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.376953125 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 14.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.9140625 + ] + ] + ], + "outputs": [ + [ + [ + 0.94140625 + ] + ], + [ + [ + 1.96875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.8828125 + ] + ], + [ + [ + 17.625 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.796875 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.8828125 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.765625 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.625 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.796875 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.5625 + ] + ], + [ + [ + 17.625 + ] + ], + [ + [ + 2.796875 + ] + ] + ], + "outputs": [ + [ + [ + 1.9140625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.6.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 7.71875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.6.feed_forward.down_proj": { + "inputs": [ + [ + [ + 8.25 + ] + ] + ], + "outputs": [ + [ + [ + 2.25 + ] + ], + [ + [ + 11.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.349609375 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 7.59375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.6796875 + ] + ] + ], + "outputs": [ + [ + [ + 2.1875 + ] + ], + [ + [ + 2.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.60546875 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.373046875 + ] + ], + [ + [ + 6.875 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4609375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.373046875 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.078125 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 6.875 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.4609375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 4.40625 + ] + ], + [ + [ + 6.875 + ] + ], + [ + [ + 1.421875 + ] + ] + ], + "outputs": [ + [ + [ + 0.625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.7.feed_forward.router": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "outputs": [ + [ + [ + 3.734375 + ] + ], + [ + [ + 28.75 + ] + ], + [ + [ + 15.375 + ] + ], + [ + [ + 50.25 + ] + ], + [ + [ + 12.625 + ] + ], + [ + [ + 23.625 + ] + ], + [ + [ + 34.0 + ] + ], + [ + [ + 21.75 + ] + ], + [ + [ + 9.5 + ] + ], + [ + [ + 31.625 + ] + ], + [ + [ + 44.25 + ] + ], + [ + [ + 34.5 + ] + ], + [ + [ + 17.75 + ] + ], + [ + [ + 9.5 + ] + ], + [ + [ + 44.75 + ] + ], + [ + [ + 15.0625 + ] + ], + [ + [ + 25.875 + ] + ] + ] + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.283203125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11572265625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08642578125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.62890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11767578125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09326171875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09326171875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.412109375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33984375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 8.5 + ] + ] + ], + "outputs": [ + [ + [ + 0.77734375 + ] + ], + [ + [ + 4.091529924525444e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.078125 + ] + ] + ], + "outputs": [ + [ + [ + 2.265625 + ] + ], + [ + [ + 3.390625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.94921875 + ] + ], + [ + [ + 16.5 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.5625 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.94921875 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.078125 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.5 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.5625 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.75 + ] + ], + [ + [ + 16.375 + ] + ], + [ + [ + 2.5625 + ] + ] + ], + "outputs": [ + [ + [ + 1.9296875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.8.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 8.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.8.feed_forward.down_proj": { + "inputs": [ + [ + [ + 12.0625 + ] + ] + ], + "outputs": [ + [ + [ + 2.84375 + ] + ], + [ + [ + 9.8125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 16.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.0 + ] + ] + ], + "outputs": [ + [ + [ + 1.609375 + ] + ], + [ + [ + 3.640625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0625 + ] + ], + [ + [ + 18.0 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.109375 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0625 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.7421875 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.0 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.109375 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.3125 + ] + ], + [ + [ + 18.0 + ] + ], + [ + [ + 3.109375 + ] + ] + ], + "outputs": [ + [ + [ + 2.0 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.9.feed_forward.router": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "outputs": [ + [ + [ + 4.5625 + ] + ], + [ + [ + 84.5 + ] + ], + [ + [ + 63.5 + ] + ], + [ + [ + 29.875 + ] + ], + [ + [ + 29.5 + ] + ], + [ + [ + 70.0 + ] + ], + [ + [ + 39.25 + ] + ], + [ + [ + 75.0 + ] + ], + [ + [ + 69.0 + ] + ], + [ + [ + 49.0 + ] + ], + [ + [ + 53.75 + ] + ], + [ + [ + 44.5 + ] + ], + [ + [ + 53.25 + ] + ], + [ + [ + 51.5 + ] + ], + [ + [ + 53.5 + ] + ], + [ + [ + 53.0 + ] + ], + [ + [ + 16.625 + ] + ] + ] + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1025390625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1083984375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10498046875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0869140625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.103515625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.111328125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08984375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08984375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 9.875 + ] + ] + ], + "outputs": [ + [ + [ + 0.70703125 + ] + ], + [ + [ + 4.911261142184431e+23 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 17.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.03125 + ] + ] + ], + "outputs": [ + [ + [ + 1.984375 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.96484375 + ] + ], + [ + [ + 19.75 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.46875 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.96484375 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.8515625 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.75 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.46875 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.4375 + ] + ], + [ + [ + 19.375 + ] + ], + [ + [ + 3.46875 + ] + ] + ], + "outputs": [ + [ + [ + 2.03125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.10.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 9.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.240234375 + ] + ] + } + }, + "language_model.model.layers.10.feed_forward.down_proj": { + "inputs": [ + [ + [ + 13.8125 + ] + ] + ], + "outputs": [ + [ + [ + 3.40625 + ] + ], + [ + [ + 12.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.48046875 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 15.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.328125 + ] + ] + ], + "outputs": [ + [ + [ + 8.625 + ] + ], + [ + [ + 8.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.77734375 + ] + ], + [ + [ + 10.0 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.46875 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.77734375 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.5 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 10.0 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.46875 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 8.625 + ] + ], + [ + [ + 10.0 + ] + ], + [ + [ + 4.46875 + ] + ] + ], + "outputs": [ + [ + [ + 2.21875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.11.feed_forward.router": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "outputs": [ + [ + [ + 4.625 + ] + ], + [ + [ + 55.5 + ] + ], + [ + [ + 57.5 + ] + ], + [ + [ + 23.375 + ] + ], + [ + [ + 53.0 + ] + ], + [ + [ + 13.6875 + ] + ], + [ + [ + 56.5 + ] + ], + [ + [ + 37.0 + ] + ], + [ + [ + 28.875 + ] + ], + [ + [ + 51.0 + ] + ], + [ + [ + 49.25 + ] + ], + [ + [ + 41.5 + ] + ], + [ + [ + 20.0 + ] + ], + [ + [ + 39.0 + ] + ], + [ + [ + 29.75 + ] + ], + [ + [ + 23.0 + ] + ], + [ + [ + 73.5 + ] + ] + ] + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10498046875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11474609375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.111328125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.41796875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 18.375 + ] + ] + ], + "outputs": [ + [ + [ + 7.46875 + ] + ], + [ + [ + 3.349654549377988e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.384765625 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 20.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.71875 + ] + ] + ], + "outputs": [ + [ + [ + 1.796875 + ] + ], + [ + [ + 3.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.89453125 + ] + ], + [ + [ + 14.25 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.0625 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.89453125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.46875 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.25 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.0625 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.8125 + ] + ], + [ + [ + 12.5 + ] + ], + [ + [ + 4.0625 + ] + ] + ], + "outputs": [ + [ + [ + 3.71875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.12.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 10.6875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.36328125 + ] + ] + } + }, + "language_model.model.layers.12.feed_forward.down_proj": { + "inputs": [ + [ + [ + 19.0 + ] + ] + ], + "outputs": [ + [ + [ + 9.25 + ] + ], + [ + [ + 20.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4453125 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.265625 + ] + ] + ], + "outputs": [ + [ + [ + 2.015625 + ] + ], + [ + [ + 6.34375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.046875 + ] + ], + [ + [ + 16.0 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.046875 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.046875 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.6484375 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.0 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.046875 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.75 + ] + ], + [ + [ + 15.75 + ] + ], + [ + [ + 3.046875 + ] + ] + ], + "outputs": [ + [ + [ + 2.265625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.13.feed_forward.router": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "outputs": [ + [ + [ + 5.5 + ] + ], + [ + [ + 60.0 + ] + ], + [ + [ + 27.125 + ] + ], + [ + [ + 57.75 + ] + ], + [ + [ + 81.5 + ] + ], + [ + [ + 46.75 + ] + ], + [ + [ + 44.75 + ] + ], + [ + [ + 63.0 + ] + ], + [ + [ + 41.75 + ] + ], + [ + [ + 58.5 + ] + ], + [ + [ + 48.0 + ] + ], + [ + [ + 76.0 + ] + ], + [ + [ + 62.75 + ] + ], + [ + [ + 69.0 + ] + ], + [ + [ + 75.5 + ] + ], + [ + [ + 42.25 + ] + ], + [ + [ + 54.0 + ] + ] + ] + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1083984375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 3.78125 + ] + ], + [ + [ + 7.476907476290152e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6015625 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 20.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.8125 + ] + ] + ], + "outputs": [ + [ + [ + 3.546875 + ] + ], + [ + [ + 5.71875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.015625 + ] + ], + [ + [ + 17.625 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.40625 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.015625 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.8125 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.625 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.40625 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.375 + ] + ], + [ + [ + 17.625 + ] + ], + [ + [ + 4.40625 + ] + ] + ], + "outputs": [ + [ + [ + 3.71875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.14.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 12.0625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.287109375 + ] + ] + } + }, + "language_model.model.layers.14.feed_forward.down_proj": { + "inputs": [ + [ + [ + 16.0 + ] + ] + ], + "outputs": [ + [ + [ + 5.375 + ] + ], + [ + [ + 20.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 23.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.921875 + ] + ] + ], + "outputs": [ + [ + [ + 20.0 + ] + ], + [ + [ + 21.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.8984375 + ] + ], + [ + [ + 14.1875 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.5 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.8984375 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.46875 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.1875 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.5 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.3125 + ] + ], + [ + [ + 14.1875 + ] + ], + [ + [ + 5.25 + ] + ] + ], + "outputs": [ + [ + [ + 3.921875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.15.feed_forward.router": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 7.375 + ] + ], + [ + [ + 33.25 + ] + ], + [ + [ + 38.5 + ] + ], + [ + [ + 71.0 + ] + ], + [ + [ + 60.25 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 91.0 + ] + ], + [ + [ + 63.0 + ] + ], + [ + [ + 89.5 + ] + ], + [ + [ + 51.25 + ] + ], + [ + [ + 70.5 + ] + ], + [ + [ + 57.75 + ] + ], + [ + [ + 61.0 + ] + ], + [ + [ + 67.5 + ] + ], + [ + [ + 26.25 + ] + ], + [ + [ + 59.75 + ] + ], + [ + [ + 66.0 + ] + ] + ] + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08837890625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08984375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1083984375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 22.125 + ] + ] + ], + "outputs": [ + [ + [ + 7.65625 + ] + ], + [ + [ + 1.5867659199682433e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.76953125 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 23.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.53125 + ] + ] + ], + "outputs": [ + [ + [ + 2.4375 + ] + ], + [ + [ + 6.3125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1015625 + ] + ], + [ + [ + 20.875 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.53125 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1015625 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.53125 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.875 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.53125 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.0625 + ] + ], + [ + [ + 20.875 + ] + ], + [ + [ + 4.125 + ] + ] + ], + "outputs": [ + [ + [ + 2.8125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.16.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 14.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.341796875 + ] + ] + } + }, + "language_model.model.layers.16.feed_forward.down_proj": { + "inputs": [ + [ + [ + 16.75 + ] + ] + ], + "outputs": [ + [ + [ + 6.65625 + ] + ], + [ + [ + 27.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8359375 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.875 + ] + ] + ], + "outputs": [ + [ + [ + 19.5 + ] + ], + [ + [ + 27.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5859375 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.078125 + ] + ], + [ + [ + 18.0 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.875 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.078125 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.40625 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.0 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.875 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.25 + ] + ], + [ + [ + 18.0 + ] + ], + [ + [ + 6.875 + ] + ] + ], + "outputs": [ + [ + [ + 6.875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.17.feed_forward.router": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 8.6875 + ] + ], + [ + [ + 58.75 + ] + ], + [ + [ + 55.0 + ] + ], + [ + [ + 44.5 + ] + ], + [ + [ + 71.0 + ] + ], + [ + [ + 49.25 + ] + ], + [ + [ + 67.5 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 67.5 + ] + ], + [ + [ + 54.5 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 61.5 + ] + ], + [ + [ + 67.5 + ] + ], + [ + [ + 51.5 + ] + ], + [ + [ + 100.5 + ] + ], + [ + [ + 69.0 + ] + ], + [ + [ + 55.0 + ] + ] + ] + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11279296875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.384765625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11767578125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11767578125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.388671875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 30.375 + ] + ] + ], + "outputs": [ + [ + [ + 3.703125 + ] + ], + [ + [ + 3.349654549377988e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33984375 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.1875 + ] + ] + ], + "outputs": [ + [ + [ + 6.625 + ] + ], + [ + [ + 12.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1015625 + ] + ], + [ + [ + 18.125 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.1875 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1015625 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.78125 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.125 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.1875 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.5 + ] + ], + [ + [ + 18.125 + ] + ], + [ + [ + 5.1875 + ] + ] + ], + "outputs": [ + [ + [ + 4.1875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.18.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 20.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.18.feed_forward.down_proj": { + "inputs": [ + [ + [ + 192.0 + ] + ] + ], + "outputs": [ + [ + [ + 8.1875 + ] + ], + [ + [ + 354.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.67578125 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 44.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.03125 + ] + ] + ], + "outputs": [ + [ + [ + 31.5 + ] + ], + [ + [ + 37.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0546875 + ] + ], + [ + [ + 13.5625 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.375 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0546875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.34375 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.5625 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.625 + ] + ], + [ + [ + 13.5625 + ] + ], + [ + [ + 11.375 + ] + ] + ], + "outputs": [ + [ + [ + 4.03125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.19.feed_forward.router": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "outputs": [ + [ + [ + 11.75 + ] + ], + [ + [ + 71.5 + ] + ], + [ + [ + 61.0 + ] + ], + [ + [ + 43.0 + ] + ], + [ + [ + 84.0 + ] + ], + [ + [ + 111.5 + ] + ], + [ + [ + 53.75 + ] + ], + [ + [ + 61.25 + ] + ], + [ + [ + 65.5 + ] + ], + [ + [ + 90.0 + ] + ], + [ + [ + 44.5 + ] + ], + [ + [ + 63.0 + ] + ], + [ + [ + 57.0 + ] + ], + [ + [ + 67.5 + ] + ], + [ + [ + 37.5 + ] + ], + [ + [ + 64.0 + ] + ], + [ + [ + 88.0 + ] + ] + ] + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.240234375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.341796875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2353515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2353515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.41796875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.39453125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 40.5 + ] + ] + ], + "outputs": [ + [ + [ + 5.0 + ] + ], + [ + [ + 1.295997295890293e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.322265625 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 53.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2490234375 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.96875 + ] + ] + ], + "outputs": [ + [ + [ + 12.8125 + ] + ], + [ + [ + 22.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34765625 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.984375 + ] + ], + [ + [ + 20.0 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.09375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.984375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.84375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.0 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.09375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.6875 + ] + ], + [ + [ + 20.0 + ] + ], + [ + [ + 7.0625 + ] + ] + ], + "outputs": [ + [ + [ + 6.96875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.20.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 42.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3671875 + ] + ] + } + }, + "language_model.model.layers.20.feed_forward.down_proj": { + "inputs": [ + [ + [ + 21.0 + ] + ] + ], + "outputs": [ + [ + [ + 10.5 + ] + ], + [ + [ + 41.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.54296875 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 58.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.859375 + ] + ] + ], + "outputs": [ + [ + [ + 26.0 + ] + ], + [ + [ + 35.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2421875 + ] + ], + [ + [ + 18.5 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.0625 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2421875 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.40625 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.0625 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.0 + ] + ], + [ + [ + 18.5 + ] + ], + [ + [ + 9.0625 + ] + ] + ], + "outputs": [ + [ + [ + 3.703125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.21.feed_forward.router": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.45703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "outputs": [ + [ + [ + 9.1875 + ] + ], + [ + [ + 52.75 + ] + ], + [ + [ + 20.0 + ] + ], + [ + [ + 102.5 + ] + ], + [ + [ + 35.0 + ] + ], + [ + [ + 73.0 + ] + ], + [ + [ + 69.0 + ] + ], + [ + [ + 33.25 + ] + ], + [ + [ + 60.5 + ] + ], + [ + [ + 46.25 + ] + ], + [ + [ + 60.5 + ] + ], + [ + [ + 69.0 + ] + ], + [ + [ + 63.75 + ] + ], + [ + [ + 42.5 + ] + ], + [ + [ + 48.0 + ] + ], + [ + [ + 77.5 + ] + ], + [ + [ + 64.5 + ] + ] + ] + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.345703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.39453125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23828125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.365234375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11474609375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.123046875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.283203125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 21.125 + ] + ] + ], + "outputs": [ + [ + [ + 6.21875 + ] + ], + [ + [ + 4.091529924525444e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.486328125 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 57.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.84375 + ] + ] + ], + "outputs": [ + [ + [ + 15.75 + ] + ], + [ + [ + 28.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1953125 + ] + ], + [ + [ + 18.375 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.78125 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1953125 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.125 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.375 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.78125 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.6875 + ] + ], + [ + [ + 18.375 + ] + ], + [ + [ + 5.53125 + ] + ] + ], + "outputs": [ + [ + [ + 3.84375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.22.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 46.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.22.feed_forward.down_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 14.875 + ] + ], + [ + [ + 77.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6796875 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 69.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.625 + ] + ] + ], + "outputs": [ + [ + [ + 70.0 + ] + ], + [ + [ + 79.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.32421875 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.09375 + ] + ], + [ + [ + 13.5 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.9375 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.09375 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.3125 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.5 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.9375 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.3125 + ] + ], + [ + [ + 13.5 + ] + ], + [ + [ + 7.9375 + ] + ] + ], + "outputs": [ + [ + [ + 5.625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.23.feed_forward.router": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.474609375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 16.375 + ] + ], + [ + [ + 41.0 + ] + ], + [ + [ + 22.25 + ] + ], + [ + [ + 54.0 + ] + ], + [ + [ + 34.75 + ] + ], + [ + [ + 44.75 + ] + ], + [ + [ + 66.0 + ] + ], + [ + [ + 93.0 + ] + ], + [ + [ + 156.0 + ] + ], + [ + [ + 53.25 + ] + ], + [ + [ + 85.0 + ] + ], + [ + [ + 140.0 + ] + ], + [ + [ + 59.0 + ] + ], + [ + [ + 29.125 + ] + ], + [ + [ + 82.5 + ] + ], + [ + [ + 62.5 + ] + ], + [ + [ + 107.5 + ] + ] + ] + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.39453125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.431640625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.478515625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11767578125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.287109375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.53125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.43359375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 51.25 + ] + ] + ], + "outputs": [ + [ + [ + 23.125 + ] + ], + [ + [ + 1.295997295890293e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.9921875 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.90625 + ] + ] + ], + "outputs": [ + [ + [ + 48.0 + ] + ], + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2734375 + ] + ], + [ + [ + 16.75 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.875 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2734375 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.53125 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.75 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.875 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.125 + ] + ], + [ + [ + 16.75 + ] + ], + [ + [ + 10.875 + ] + ] + ], + "outputs": [ + [ + [ + 6.90625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.24.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.24.feed_forward.down_proj": { + "inputs": [ + [ + [ + 25.375 + ] + ] + ], + "outputs": [ + [ + [ + 21.75 + ] + ], + [ + [ + 100.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.703125 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.28125 + ] + ] + ], + "outputs": [ + [ + [ + 53.0 + ] + ], + [ + [ + 69.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1953125 + ] + ], + [ + [ + 20.75 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.0 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1953125 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.3125 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.75 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.0 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.3125 + ] + ], + [ + [ + 20.625 + ] + ], + [ + [ + 11.0 + ] + ] + ], + "outputs": [ + [ + [ + 7.28125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.25.feed_forward.router": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.412109375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "outputs": [ + [ + [ + 7.15625 + ] + ], + [ + [ + 69.0 + ] + ], + [ + [ + 86.0 + ] + ], + [ + [ + 44.0 + ] + ], + [ + [ + 41.25 + ] + ], + [ + [ + 69.5 + ] + ], + [ + [ + 54.25 + ] + ], + [ + [ + 58.25 + ] + ], + [ + [ + 38.0 + ] + ], + [ + [ + 62.75 + ] + ], + [ + [ + 28.0 + ] + ], + [ + [ + 51.25 + ] + ], + [ + [ + 28.0 + ] + ], + [ + [ + 50.25 + ] + ], + [ + [ + 43.5 + ] + ], + [ + [ + 91.0 + ] + ], + [ + [ + 47.75 + ] + ] + ] + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.357421875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.419921875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 37.5 + ] + ] + ], + "outputs": [ + [ + [ + 33.5 + ] + ], + [ + [ + 3.800761300447494e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8671875 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 63.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.0625 + ] + ] + ], + "outputs": [ + [ + [ + 30.875 + ] + ], + [ + [ + 48.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.296875 + ] + ], + [ + [ + 19.0 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.375 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.296875 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.40625 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.0 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.375 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.25 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 13.375 + ] + ] + ], + "outputs": [ + [ + [ + 7.0625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.26.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 45.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.62890625 + ] + ] + } + }, + "language_model.model.layers.26.feed_forward.down_proj": { + "inputs": [ + [ + [ + 21.625 + ] + ] + ], + "outputs": [ + [ + [ + 22.125 + ] + ], + [ + [ + 113.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.87109375 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.34375 + ] + ] + ], + "outputs": [ + [ + [ + 60.25 + ] + ], + [ + [ + 71.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.99609375 + ] + ], + [ + [ + 15.625 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.75 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.99609375 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.5625 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.625 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.75 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.9375 + ] + ], + [ + [ + 15.625 + ] + ], + [ + [ + 11.75 + ] + ] + ], + "outputs": [ + [ + [ + 7.34375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.27.feed_forward.router": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "outputs": [ + [ + [ + 10.0 + ] + ], + [ + [ + 47.25 + ] + ], + [ + [ + 94.5 + ] + ], + [ + [ + 39.5 + ] + ], + [ + [ + 33.25 + ] + ], + [ + [ + 91.0 + ] + ], + [ + [ + 47.0 + ] + ], + [ + [ + 92.0 + ] + ], + [ + [ + 51.5 + ] + ], + [ + [ + 49.75 + ] + ], + [ + [ + 24.5 + ] + ], + [ + [ + 44.0 + ] + ], + [ + [ + 68.0 + ] + ], + [ + [ + 56.75 + ] + ], + [ + [ + 78.0 + ] + ], + [ + [ + 56.5 + ] + ], + [ + [ + 131.0 + ] + ] + ] + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08935546875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08984375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08935546875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1005859375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.416015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 37.0 + ] + ] + ], + "outputs": [ + [ + [ + 22.0 + ] + ], + [ + [ + 4.091529924525444e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.60546875 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 78.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.1875 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 97.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.15625 + ] + ], + [ + [ + 18.125 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.125 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.15625 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.0 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.125 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.125 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.1875 + ] + ], + [ + [ + 18.0 + ] + ], + [ + [ + 9.125 + ] + ] + ], + "outputs": [ + [ + [ + 8.1875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.28.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3984375 + ] + ] + } + }, + "language_model.model.layers.28.feed_forward.down_proj": { + "inputs": [ + [ + [ + 33.5 + ] + ] + ], + "outputs": [ + [ + [ + 140.0 + ] + ], + [ + [ + 149.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.640625 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 67.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.25 + ] + ] + ], + "outputs": [ + [ + [ + 78.0 + ] + ], + [ + [ + 84.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0625 + ] + ], + [ + [ + 18.125 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.5625 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0625 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.28125 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.125 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.5625 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.875 + ] + ], + [ + [ + 17.875 + ] + ], + [ + [ + 8.5625 + ] + ] + ], + "outputs": [ + [ + [ + 7.25 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.29.feed_forward.router": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.333984375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "outputs": [ + [ + [ + 15.0625 + ] + ], + [ + [ + 43.75 + ] + ], + [ + [ + 55.0 + ] + ], + [ + [ + 77.5 + ] + ], + [ + [ + 47.5 + ] + ], + [ + [ + 64.5 + ] + ], + [ + [ + 50.75 + ] + ], + [ + [ + 92.0 + ] + ], + [ + [ + 69.5 + ] + ], + [ + [ + 72.5 + ] + ], + [ + [ + 135.0 + ] + ], + [ + [ + 103.5 + ] + ], + [ + [ + 82.5 + ] + ], + [ + [ + 97.5 + ] + ], + [ + [ + 88.0 + ] + ], + [ + [ + 63.0 + ] + ], + [ + [ + 66.5 + ] + ] + ] + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.46484375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.412109375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09326171875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.287109375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3828125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 52.75 + ] + ] + ], + "outputs": [ + [ + [ + 11.25 + ] + ], + [ + [ + 3.349654549377988e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.93359375 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.21875 + ] + ] + ], + "outputs": [ + [ + [ + 50.25 + ] + ], + [ + [ + 56.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.4609375 + ] + ], + [ + [ + 20.125 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.3125 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4609375 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.875 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.125 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.3125 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 18.5 + ] + ], + [ + [ + 20.125 + ] + ], + [ + [ + 8.3125 + ] + ] + ], + "outputs": [ + [ + [ + 7.21875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.30.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 32.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37109375 + ] + ] + } + }, + "language_model.model.layers.30.feed_forward.down_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "outputs": [ + [ + [ + 30.25 + ] + ], + [ + [ + 159.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.58984375 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.25 + ] + ] + ], + "outputs": [ + [ + [ + 77.0 + ] + ], + [ + [ + 115.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.046875 + ] + ], + [ + [ + 14.6875 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.0625 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.046875 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.75 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.6875 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.0625 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.5 + ] + ], + [ + [ + 14.6875 + ] + ], + [ + [ + 10.0625 + ] + ] + ], + "outputs": [ + [ + [ + 8.25 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.31.feed_forward.router": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "outputs": [ + [ + [ + 18.25 + ] + ], + [ + [ + 83.5 + ] + ], + [ + [ + 91.0 + ] + ], + [ + [ + 56.25 + ] + ], + [ + [ + 86.5 + ] + ], + [ + [ + 235.0 + ] + ], + [ + [ + 81.0 + ] + ], + [ + [ + 108.5 + ] + ], + [ + [ + 34.5 + ] + ], + [ + [ + 101.5 + ] + ], + [ + [ + 84.5 + ] + ], + [ + [ + 145.0 + ] + ], + [ + [ + 73.0 + ] + ], + [ + [ + 127.5 + ] + ], + [ + [ + 88.5 + ] + ], + [ + [ + 34.0 + ] + ], + [ + [ + 60.25 + ] + ] + ] + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.244140625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33203125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33203125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.322265625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 149.0 + ] + ] + ], + "outputs": [ + [ + [ + 105.5 + ] + ], + [ + [ + 2.928455428213643e+36 + ] + ] + ], + "params": { + "weight": [ + [ + 0.75 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 53.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.65625 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 109.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.5546875 + ] + ], + [ + [ + 16.625 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.4375 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.5546875 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.0 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.4375 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 18.875 + ] + ], + [ + [ + 16.625 + ] + ], + [ + [ + 8.4375 + ] + ] + ], + "outputs": [ + [ + [ + 6.65625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.32.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 32.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.400390625 + ] + ] + } + }, + "language_model.model.layers.32.feed_forward.down_proj": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "outputs": [ + [ + [ + 29.375 + ] + ], + [ + [ + 75.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.474609375 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 44.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.71875 + ] + ] + ], + "outputs": [ + [ + [ + 94.0 + ] + ], + [ + [ + 132.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.384765625 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2734375 + ] + ], + [ + [ + 18.75 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.75 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2734375 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.4375 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.75 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.75 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 17.125 + ] + ], + [ + [ + 18.75 + ] + ], + [ + [ + 9.75 + ] + ] + ], + "outputs": [ + [ + [ + 7.71875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.33.feed_forward.router": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "outputs": [ + [ + [ + 37.25 + ] + ], + [ + [ + 64.5 + ] + ], + [ + [ + 114.0 + ] + ], + [ + [ + 95.0 + ] + ], + [ + [ + 112.0 + ] + ], + [ + [ + 87.0 + ] + ], + [ + [ + 90.0 + ] + ], + [ + [ + 116.0 + ] + ], + [ + [ + 77.0 + ] + ], + [ + [ + 97.0 + ] + ], + [ + [ + 101.5 + ] + ], + [ + [ + 87.0 + ] + ], + [ + [ + 166.0 + ] + ], + [ + [ + 76.5 + ] + ], + [ + [ + 150.0 + ] + ], + [ + [ + 132.0 + ] + ], + [ + [ + 163.0 + ] + ] + ] + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.244140625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.431640625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 69.0 + ] + ] + ], + "outputs": [ + [ + [ + 66.5 + ] + ], + [ + [ + 1.5867659199682433e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7109375 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 54.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.25 + ] + ] + ], + "outputs": [ + [ + [ + 55.75 + ] + ], + [ + [ + 98.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.09375 + ] + ], + [ + [ + 21.125 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.25 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.09375 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.03125 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 21.125 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.25 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.5 + ] + ], + [ + [ + 21.125 + ] + ], + [ + [ + 8.25 + ] + ] + ], + "outputs": [ + [ + [ + 7.25 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.34.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.50390625 + ] + ] + } + }, + "language_model.model.layers.34.feed_forward.down_proj": { + "inputs": [ + [ + [ + 104.0 + ] + ] + ], + "outputs": [ + [ + [ + 36.5 + ] + ], + [ + [ + 162.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6640625 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 54.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.75 + ] + ] + ], + "outputs": [ + [ + [ + 110.5 + ] + ], + [ + [ + 116.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.078125 + ] + ], + [ + [ + 18.125 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.3125 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.078125 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.65625 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.125 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.3125 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.1875 + ] + ], + [ + [ + 18.125 + ] + ], + [ + [ + 8.3125 + ] + ] + ], + "outputs": [ + [ + [ + 7.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.35.feed_forward.router": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "outputs": [ + [ + [ + 24.375 + ] + ], + [ + [ + 109.5 + ] + ], + [ + [ + 111.0 + ] + ], + [ + [ + 115.5 + ] + ], + [ + [ + 87.0 + ] + ], + [ + [ + 180.0 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 131.0 + ] + ], + [ + [ + 46.5 + ] + ], + [ + [ + 51.0 + ] + ], + [ + [ + 105.5 + ] + ], + [ + [ + 63.5 + ] + ], + [ + [ + 76.0 + ] + ], + [ + [ + 140.0 + ] + ], + [ + [ + 135.0 + ] + ], + [ + [ + 197.0 + ] + ], + [ + [ + 47.75 + ] + ] + ] + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.54296875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.287109375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.38671875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.373046875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.421875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1044921875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34765625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 96.5 + ] + ] + ], + "outputs": [ + [ + [ + 101.5 + ] + ], + [ + [ + 7.476907476290152e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.90234375 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 56.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.o_proj": { + "inputs": [ + [ + [ + 9.4375 + ] + ] + ], + "outputs": [ + [ + [ + 36.75 + ] + ], + [ + [ + 57.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.396484375 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1796875 + ] + ], + [ + [ + 21.875 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.75 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1796875 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.0625 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 21.875 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.75 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.625 + ] + ], + [ + [ + 21.875 + ] + ], + [ + [ + 9.75 + ] + ] + ], + "outputs": [ + [ + [ + 9.4375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.36.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 29.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.72265625 + ] + ] + } + }, + "language_model.model.layers.36.feed_forward.down_proj": { + "inputs": [ + [ + [ + 77.5 + ] + ] + ], + "outputs": [ + [ + [ + 43.0 + ] + ], + [ + [ + 94.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.859375 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.o_proj": { + "inputs": [ + [ + [ + 10.5625 + ] + ] + ], + "outputs": [ + [ + [ + 97.5 + ] + ], + [ + [ + 150.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 2.171875 + ] + ], + [ + [ + 22.25 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.9375 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.171875 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.875 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 22.25 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 14.9375 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 24.875 + ] + ], + [ + [ + 22.0 + ] + ], + [ + [ + 14.9375 + ] + ] + ], + "outputs": [ + [ + [ + 10.5625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.37.feed_forward.router": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "outputs": [ + [ + [ + 28.25 + ] + ], + [ + [ + 84.0 + ] + ], + [ + [ + 177.0 + ] + ], + [ + [ + 166.0 + ] + ], + [ + [ + 132.0 + ] + ], + [ + [ + 157.0 + ] + ], + [ + [ + 116.0 + ] + ], + [ + [ + 133.0 + ] + ], + [ + [ + 78.0 + ] + ], + [ + [ + 131.0 + ] + ], + [ + [ + 141.0 + ] + ], + [ + [ + 155.0 + ] + ], + [ + [ + 50.0 + ] + ], + [ + [ + 55.25 + ] + ], + [ + [ + 164.0 + ] + ], + [ + [ + 98.5 + ] + ], + [ + [ + 85.0 + ] + ] + ] + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.091796875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.431640625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10009765625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.376953125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33984375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 123.5 + ] + ] + ], + "outputs": [ + [ + [ + 19.625 + ] + ], + [ + [ + 3.349654549377988e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4453125 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 47.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.o_proj": { + "inputs": [ + [ + [ + 9.9375 + ] + ] + ], + "outputs": [ + [ + [ + 29.75 + ] + ], + [ + [ + 66.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.91796875 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.40625 + ] + ], + [ + [ + 14.1875 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.125 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.40625 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.3125 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.1875 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.125 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 17.75 + ] + ], + [ + [ + 14.125 + ] + ], + [ + [ + 10.125 + ] + ] + ], + "outputs": [ + [ + [ + 9.9375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.38.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 31.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.369140625 + ] + ] + } + }, + "language_model.model.layers.38.feed_forward.down_proj": { + "inputs": [ + [ + [ + 103.5 + ] + ] + ], + "outputs": [ + [ + [ + 31.125 + ] + ], + [ + [ + 262.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.67578125 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 52.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.84375 + ] + ] + ], + "outputs": [ + [ + [ + 126.5 + ] + ], + [ + [ + 154.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.359375 + ] + ], + [ + [ + 10.375 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 15.6875 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.359375 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.5625 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 10.375 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 15.6875 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.625 + ] + ], + [ + [ + 10.375 + ] + ], + [ + [ + 15.6875 + ] + ] + ], + "outputs": [ + [ + [ + 7.84375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.39.feed_forward.router": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "outputs": [ + [ + [ + 27.625 + ] + ], + [ + [ + 98.0 + ] + ], + [ + [ + 171.0 + ] + ], + [ + [ + 195.0 + ] + ], + [ + [ + 93.0 + ] + ], + [ + [ + 83.0 + ] + ], + [ + [ + 137.0 + ] + ], + [ + [ + 151.0 + ] + ], + [ + [ + 143.0 + ] + ], + [ + [ + 98.5 + ] + ], + [ + [ + 185.0 + ] + ], + [ + [ + 173.0 + ] + ], + [ + [ + 147.0 + ] + ], + [ + [ + 199.0 + ] + ], + [ + [ + 107.5 + ] + ], + [ + [ + 131.0 + ] + ], + [ + [ + 116.0 + ] + ] + ] + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11474609375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2197265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 136.0 + ] + ] + ], + "outputs": [ + [ + [ + 14.9375 + ] + ], + [ + [ + 3.349654549377988e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.388671875 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 46.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.o_proj": { + "inputs": [ + [ + [ + 13.0625 + ] + ] + ], + "outputs": [ + [ + [ + 94.0 + ] + ], + [ + [ + 102.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.365234375 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.4453125 + ] + ], + [ + [ + 17.625 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.875 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4453125 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.75 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.625 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.875 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.125 + ] + ], + [ + [ + 17.125 + ] + ], + [ + [ + 13.875 + ] + ] + ], + "outputs": [ + [ + [ + 13.0625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.40.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3046875 + ] + ] + } + }, + "language_model.model.layers.40.feed_forward.down_proj": { + "inputs": [ + [ + [ + 149.0 + ] + ] + ], + "outputs": [ + [ + [ + 28.5 + ] + ], + [ + [ + 127.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.451171875 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 35.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.34375 + ] + ] + ], + "outputs": [ + [ + [ + 160.0 + ] + ], + [ + [ + 177.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.7578125 + ] + ], + [ + [ + 17.0 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.6875 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.7578125 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.34375 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.0 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.6875 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 21.25 + ] + ], + [ + [ + 16.625 + ] + ], + [ + [ + 10.6875 + ] + ] + ], + "outputs": [ + [ + [ + 7.34375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.41.feed_forward.router": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "outputs": [ + [ + [ + 52.75 + ] + ], + [ + [ + 120.0 + ] + ], + [ + [ + 173.0 + ] + ], + [ + [ + 146.0 + ] + ], + [ + [ + 71.5 + ] + ], + [ + [ + 184.0 + ] + ], + [ + [ + 128.0 + ] + ], + [ + [ + 176.0 + ] + ], + [ + [ + 123.5 + ] + ], + [ + [ + 121.5 + ] + ], + [ + [ + 83.0 + ] + ], + [ + [ + 146.0 + ] + ], + [ + [ + 156.0 + ] + ], + [ + [ + 104.0 + ] + ], + [ + [ + 182.0 + ] + ], + [ + [ + 237.0 + ] + ], + [ + [ + 226.0 + ] + ] + ] + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.376953125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12255859375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.205078125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.345703125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3203125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11083984375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.41015625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.498046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.330078125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.421875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 201.0 + ] + ] + ], + "outputs": [ + [ + [ + 23.5 + ] + ], + [ + [ + 7.476907476290152e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4609375 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 39.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.o_proj": { + "inputs": [ + [ + [ + 13.1875 + ] + ] + ], + "outputs": [ + [ + [ + 65.0 + ] + ], + [ + [ + 146.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.87109375 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.78125 + ] + ], + [ + [ + 18.875 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.75 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.78125 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.875 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.875 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.75 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 20.5 + ] + ], + [ + [ + 18.125 + ] + ], + [ + [ + 13.75 + ] + ] + ], + "outputs": [ + [ + [ + 13.1875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.42.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 43.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.310546875 + ] + ] + } + }, + "language_model.model.layers.42.feed_forward.down_proj": { + "inputs": [ + [ + [ + 224.0 + ] + ] + ], + "outputs": [ + [ + [ + 70.5 + ] + ], + [ + [ + 218.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.765625 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 43.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.o_proj": { + "inputs": [ + [ + [ + 14.75 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 232.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2421875 + ] + ], + [ + [ + 16.125 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 16.375 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2421875 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.125 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.125 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 16.375 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.9375 + ] + ], + [ + [ + 16.125 + ] + ], + [ + [ + 16.375 + ] + ] + ], + "outputs": [ + [ + [ + 14.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.43.feed_forward.router": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "outputs": [ + [ + [ + 51.0 + ] + ], + [ + [ + 76.5 + ] + ], + [ + [ + 241.0 + ] + ], + [ + [ + 171.0 + ] + ], + [ + [ + 200.0 + ] + ], + [ + [ + 124.0 + ] + ], + [ + [ + 96.0 + ] + ], + [ + [ + 256.0 + ] + ], + [ + [ + 163.0 + ] + ], + [ + [ + 290.0 + ] + ], + [ + [ + 149.0 + ] + ], + [ + [ + 163.0 + ] + ], + [ + [ + 145.0 + ] + ], + [ + [ + 118.5 + ] + ], + [ + [ + 260.0 + ] + ], + [ + [ + 256.0 + ] + ], + [ + [ + 215.0 + ] + ] + ] + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11474609375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.251953125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 290.0 + ] + ] + ], + "outputs": [ + [ + [ + 27.875 + ] + ], + [ + [ + 1.295997295890293e+38 + ] + ] + ], + "params": { + "weight": [ + [ + 0.439453125 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 29.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.o_proj": { + "inputs": [ + [ + [ + 15.5 + ] + ] + ], + "outputs": [ + [ + [ + 81.0 + ] + ], + [ + [ + 170.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.4140625 + ] + ], + [ + [ + 21.125 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 17.75 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4140625 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 15.1875 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 21.125 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 17.75 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.3125 + ] + ], + [ + [ + 21.125 + ] + ], + [ + [ + 17.75 + ] + ] + ], + "outputs": [ + [ + [ + 15.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.44.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 42.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.44.feed_forward.down_proj": { + "inputs": [ + [ + [ + 344.0 + ] + ] + ], + "outputs": [ + [ + [ + 75.5 + ] + ], + [ + [ + 200.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.640625 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 28.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.o_proj": { + "inputs": [ + [ + [ + 17.5 + ] + ] + ], + "outputs": [ + [ + [ + 170.0 + ] + ], + [ + [ + 258.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.6796875 + ] + ], + [ + [ + 20.0 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 22.125 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.6796875 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.3125 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.0 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 22.125 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 19.25 + ] + ], + [ + [ + 19.5 + ] + ], + [ + [ + 22.125 + ] + ] + ], + "outputs": [ + [ + [ + 17.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.45.feed_forward.router": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "outputs": [ + [ + [ + 67.5 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 100.5 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 62.0 + ] + ], + [ + [ + 155.0 + ] + ], + [ + [ + 59.75 + ] + ], + [ + [ + 280.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 105.5 + ] + ], + [ + [ + 207.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 142.0 + ] + ], + [ + [ + 0.0 + ] + ] + ] + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3203125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11279296875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.455078125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11376953125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.310546875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 300.0 + ] + ] + ], + "outputs": [ + [ + [ + 37.5 + ] + ], + [ + [ + 1.5784582449945876e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 27.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.o_proj": { + "inputs": [ + [ + [ + 21.0 + ] + ] + ], + "outputs": [ + [ + [ + 126.0 + ] + ], + [ + [ + 221.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.6796875 + ] + ], + [ + [ + 18.0 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 21.75 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.6796875 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 18.25 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.0 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 21.75 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 20.125 + ] + ], + [ + [ + 18.0 + ] + ], + [ + [ + 21.75 + ] + ] + ], + "outputs": [ + [ + [ + 21.0 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.46.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 40.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.55859375 + ] + ] + } + }, + "language_model.model.layers.46.feed_forward.down_proj": { + "inputs": [ + [ + [ + 328.0 + ] + ] + ], + "outputs": [ + [ + [ + 1680.0 + ] + ], + [ + [ + 2400.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5859375 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.65625 + ] + ] + ], + "outputs": [ + [ + [ + 96.5 + ] + ], + [ + [ + 125.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.515625 + ] + ], + [ + [ + 12.3125 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.9375 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.515625 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.65625 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 12.3125 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.9375 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 6.09375 + ] + ], + [ + [ + 12.3125 + ] + ], + [ + [ + 9.9375 + ] + ] + ], + "outputs": [ + [ + [ + 7.65625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.47.feed_forward.router": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "outputs": [ + [ + [ + 1020.0 + ] + ], + [ + [ + 452.0 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 69.5 + ] + ], + [ + [ + 245.0 + ] + ], + [ + [ + 155.0 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 206.0 + ] + ], + [ + [ + 67.5 + ] + ], + [ + [ + 384.0 + ] + ], + [ + [ + 132.0 + ] + ], + [ + [ + 54.75 + ] + ], + [ + [ + 676.0 + ] + ], + [ + [ + 200.0 + ] + ], + [ + [ + 214.0 + ] + ], + [ + [ + 286.0 + ] + ], + [ + [ + 360.0 + ] + ] + ] + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23828125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7421875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.55859375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.93359375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.484375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.73828125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.53515625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.59765625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.61328125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.9296875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.765625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.96875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5078125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.73828125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.390625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 2432.0 + ] + ] + ], + "outputs": [ + [ + [ + 1240.0 + ] + ], + [ + [ + 7.476907476290152e+37 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40625 + ] + ] + } + }, + "language_model.lm_head": { + "inputs": [ + [ + [ + 54.25 + ] + ] + ], + "params": { + "weight": [ + [ + 1.28125 + ] + ] + } + } + } +} \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_6_8.npz b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_6_8.npz new file mode 100644 index 000000000000..5baeb8637276 Binary files /dev/null and b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_6_8.npz differ diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_6_8_mod_list.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_6_8_mod_list.json new file mode 100644 index 000000000000..449fce78b77d --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_6_8_mod_list.json @@ -0,0 +1,1559 @@ +[ + "vision_model.patch_embedding.linear", + "vision_model.model.layers.0.self_attn.qkv_proj", + "vision_model.model.layers.0.self_attn.o_proj", + "vision_model.model.layers.0.mlp.fc1", + "vision_model.model.layers.0.mlp.fc2", + "vision_model.model.layers.1.self_attn.qkv_proj", + "vision_model.model.layers.1.self_attn.o_proj", + "vision_model.model.layers.1.mlp.fc1", + "vision_model.model.layers.1.mlp.fc2", + "vision_model.model.layers.2.self_attn.qkv_proj", + "vision_model.model.layers.2.self_attn.o_proj", + "vision_model.model.layers.2.mlp.fc1", + "vision_model.model.layers.2.mlp.fc2", + "vision_model.model.layers.3.self_attn.qkv_proj", + "vision_model.model.layers.3.self_attn.o_proj", + "vision_model.model.layers.3.mlp.fc1", + "vision_model.model.layers.3.mlp.fc2", + "vision_model.model.layers.4.self_attn.qkv_proj", + "vision_model.model.layers.4.self_attn.o_proj", + "vision_model.model.layers.4.mlp.fc1", + "vision_model.model.layers.4.mlp.fc2", + "vision_model.model.layers.5.self_attn.qkv_proj", + "vision_model.model.layers.5.self_attn.o_proj", + "vision_model.model.layers.5.mlp.fc1", + "vision_model.model.layers.5.mlp.fc2", + "vision_model.model.layers.6.self_attn.qkv_proj", + "vision_model.model.layers.6.self_attn.o_proj", + "vision_model.model.layers.6.mlp.fc1", + "vision_model.model.layers.6.mlp.fc2", + "vision_model.model.layers.7.self_attn.qkv_proj", + "vision_model.model.layers.7.self_attn.o_proj", + "vision_model.model.layers.7.mlp.fc1", + "vision_model.model.layers.7.mlp.fc2", + "vision_model.model.layers.8.self_attn.qkv_proj", + "vision_model.model.layers.8.self_attn.o_proj", + "vision_model.model.layers.8.mlp.fc1", + "vision_model.model.layers.8.mlp.fc2", + "vision_model.model.layers.9.self_attn.qkv_proj", + "vision_model.model.layers.9.self_attn.o_proj", + "vision_model.model.layers.9.mlp.fc1", + "vision_model.model.layers.9.mlp.fc2", + "vision_model.model.layers.10.self_attn.qkv_proj", + "vision_model.model.layers.10.self_attn.o_proj", + "vision_model.model.layers.10.mlp.fc1", + "vision_model.model.layers.10.mlp.fc2", + "vision_model.model.layers.11.self_attn.qkv_proj", + "vision_model.model.layers.11.self_attn.o_proj", + "vision_model.model.layers.11.mlp.fc1", + "vision_model.model.layers.11.mlp.fc2", + "vision_model.model.layers.12.self_attn.qkv_proj", + "vision_model.model.layers.12.self_attn.o_proj", + "vision_model.model.layers.12.mlp.fc1", + "vision_model.model.layers.12.mlp.fc2", + "vision_model.model.layers.13.self_attn.qkv_proj", + "vision_model.model.layers.13.self_attn.o_proj", + "vision_model.model.layers.13.mlp.fc1", + "vision_model.model.layers.13.mlp.fc2", + "vision_model.model.layers.14.self_attn.qkv_proj", + "vision_model.model.layers.14.self_attn.o_proj", + "vision_model.model.layers.14.mlp.fc1", + "vision_model.model.layers.14.mlp.fc2", + "vision_model.model.layers.15.self_attn.qkv_proj", + "vision_model.model.layers.15.self_attn.o_proj", + "vision_model.model.layers.15.mlp.fc1", + "vision_model.model.layers.15.mlp.fc2", + "vision_model.model.layers.16.self_attn.qkv_proj", + "vision_model.model.layers.16.self_attn.o_proj", + "vision_model.model.layers.16.mlp.fc1", + "vision_model.model.layers.16.mlp.fc2", + "vision_model.model.layers.17.self_attn.qkv_proj", + "vision_model.model.layers.17.self_attn.o_proj", + "vision_model.model.layers.17.mlp.fc1", + "vision_model.model.layers.17.mlp.fc2", + "vision_model.model.layers.18.self_attn.qkv_proj", + "vision_model.model.layers.18.self_attn.o_proj", + "vision_model.model.layers.18.mlp.fc1", + "vision_model.model.layers.18.mlp.fc2", + "vision_model.model.layers.19.self_attn.qkv_proj", + "vision_model.model.layers.19.self_attn.o_proj", + "vision_model.model.layers.19.mlp.fc1", + "vision_model.model.layers.19.mlp.fc2", + "vision_model.model.layers.20.self_attn.qkv_proj", + "vision_model.model.layers.20.self_attn.o_proj", + "vision_model.model.layers.20.mlp.fc1", + "vision_model.model.layers.20.mlp.fc2", + "vision_model.model.layers.21.self_attn.qkv_proj", + "vision_model.model.layers.21.self_attn.o_proj", + "vision_model.model.layers.21.mlp.fc1", + "vision_model.model.layers.21.mlp.fc2", + "vision_model.model.layers.22.self_attn.qkv_proj", + "vision_model.model.layers.22.self_attn.o_proj", + "vision_model.model.layers.22.mlp.fc1", + "vision_model.model.layers.22.mlp.fc2", + "vision_model.model.layers.23.self_attn.qkv_proj", + "vision_model.model.layers.23.self_attn.o_proj", + "vision_model.model.layers.23.mlp.fc1", + "vision_model.model.layers.23.mlp.fc2", + "vision_model.model.layers.24.self_attn.qkv_proj", + "vision_model.model.layers.24.self_attn.o_proj", + "vision_model.model.layers.24.mlp.fc1", + "vision_model.model.layers.24.mlp.fc2", + "vision_model.model.layers.25.self_attn.qkv_proj", + "vision_model.model.layers.25.self_attn.o_proj", + "vision_model.model.layers.25.mlp.fc1", + "vision_model.model.layers.25.mlp.fc2", + "vision_model.model.layers.26.self_attn.qkv_proj", + "vision_model.model.layers.26.self_attn.o_proj", + "vision_model.model.layers.26.mlp.fc1", + "vision_model.model.layers.26.mlp.fc2", + "vision_model.model.layers.27.self_attn.qkv_proj", + "vision_model.model.layers.27.self_attn.o_proj", + "vision_model.model.layers.27.mlp.fc1", + "vision_model.model.layers.27.mlp.fc2", + "vision_model.model.layers.28.self_attn.qkv_proj", + "vision_model.model.layers.28.self_attn.o_proj", + "vision_model.model.layers.28.mlp.fc1", + "vision_model.model.layers.28.mlp.fc2", + "vision_model.model.layers.29.self_attn.qkv_proj", + "vision_model.model.layers.29.self_attn.o_proj", + "vision_model.model.layers.29.mlp.fc1", + "vision_model.model.layers.29.mlp.fc2", + "vision_model.model.layers.30.self_attn.qkv_proj", + "vision_model.model.layers.30.self_attn.o_proj", + "vision_model.model.layers.30.mlp.fc1", + "vision_model.model.layers.30.mlp.fc2", + "vision_model.model.layers.31.self_attn.qkv_proj", + "vision_model.model.layers.31.self_attn.o_proj", + "vision_model.model.layers.31.mlp.fc1", + "vision_model.model.layers.31.mlp.fc2", + "vision_model.model.layers.32.self_attn.qkv_proj", + "vision_model.model.layers.32.self_attn.o_proj", + "vision_model.model.layers.32.mlp.fc1", + "vision_model.model.layers.32.mlp.fc2", + "vision_model.model.layers.33.self_attn.qkv_proj", + "vision_model.model.layers.33.self_attn.o_proj", + "vision_model.model.layers.33.mlp.fc1", + "vision_model.model.layers.33.mlp.fc2", + "vision_model.vision_adapter.mlp.fc1", + "vision_model.vision_adapter.mlp.fc2", + "multi_modal_projector.linear_1", + "language_model.model.layers.0.self_attn.qkv_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.0.self_attn.attn.impl.softmax", + "language_model.model.layers.0.self_attn.attn.impl.matmul_av", + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.0.self_attn.attn.impl.k_cache", + "language_model.model.layers.0.self_attn.attn.impl.v_cache", + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.0.feed_forward.gate_up_proj", + "language_model.model.layers.0.feed_forward.down_proj", + "language_model.model.layers.1.self_attn.qkv_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.1.self_attn.attn.impl.softmax", + "language_model.model.layers.1.self_attn.attn.impl.matmul_av", + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.1.self_attn.attn.impl.k_cache", + "language_model.model.layers.1.self_attn.attn.impl.v_cache", + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.1.feed_forward.router", + "language_model.model.layers.1.feed_forward.experts", + "language_model.model.layers.1.feed_forward.experts.moe_op", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.1.feed_forward.shared_expert.down_proj", + "language_model.model.layers.2.self_attn.qkv_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.2.self_attn.attn.impl.softmax", + "language_model.model.layers.2.self_attn.attn.impl.matmul_av", + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.2.self_attn.attn.impl.k_cache", + "language_model.model.layers.2.self_attn.attn.impl.v_cache", + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.2.feed_forward.gate_up_proj", + "language_model.model.layers.2.feed_forward.down_proj", + "language_model.model.layers.3.self_attn.qkv_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.3.self_attn.attn.impl.softmax", + "language_model.model.layers.3.self_attn.attn.impl.matmul_av", + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.3.self_attn.attn.impl.k_cache", + "language_model.model.layers.3.self_attn.attn.impl.v_cache", + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.3.feed_forward.router", + "language_model.model.layers.3.feed_forward.experts", + "language_model.model.layers.3.feed_forward.experts.moe_op", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.3.feed_forward.shared_expert.down_proj", + "language_model.model.layers.4.self_attn.qkv_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.4.self_attn.attn.impl.softmax", + "language_model.model.layers.4.self_attn.attn.impl.matmul_av", + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.4.self_attn.attn.impl.k_cache", + "language_model.model.layers.4.self_attn.attn.impl.v_cache", + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.4.feed_forward.gate_up_proj", + "language_model.model.layers.4.feed_forward.down_proj", + "language_model.model.layers.5.self_attn.qkv_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.5.self_attn.attn.impl.softmax", + "language_model.model.layers.5.self_attn.attn.impl.matmul_av", + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.5.self_attn.attn.impl.k_cache", + "language_model.model.layers.5.self_attn.attn.impl.v_cache", + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.5.feed_forward.router", + "language_model.model.layers.5.feed_forward.experts", + "language_model.model.layers.5.feed_forward.experts.moe_op", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.5.feed_forward.shared_expert.down_proj", + "language_model.model.layers.6.self_attn.qkv_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.6.self_attn.attn.impl.softmax", + "language_model.model.layers.6.self_attn.attn.impl.matmul_av", + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.6.self_attn.attn.impl.k_cache", + "language_model.model.layers.6.self_attn.attn.impl.v_cache", + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.6.feed_forward.gate_up_proj", + "language_model.model.layers.6.feed_forward.down_proj", + "language_model.model.layers.7.self_attn.qkv_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.7.self_attn.attn.impl.softmax", + "language_model.model.layers.7.self_attn.attn.impl.matmul_av", + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.7.self_attn.attn.impl.k_cache", + "language_model.model.layers.7.self_attn.attn.impl.v_cache", + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.7.feed_forward.router", + "language_model.model.layers.7.feed_forward.experts", + "language_model.model.layers.7.feed_forward.experts.moe_op", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.7.feed_forward.shared_expert.down_proj", + "language_model.model.layers.8.self_attn.qkv_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.8.self_attn.attn.impl.softmax", + "language_model.model.layers.8.self_attn.attn.impl.matmul_av", + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.8.self_attn.attn.impl.k_cache", + "language_model.model.layers.8.self_attn.attn.impl.v_cache", + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.8.feed_forward.gate_up_proj", + "language_model.model.layers.8.feed_forward.down_proj", + "language_model.model.layers.9.self_attn.qkv_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.9.self_attn.attn.impl.softmax", + "language_model.model.layers.9.self_attn.attn.impl.matmul_av", + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.9.self_attn.attn.impl.k_cache", + "language_model.model.layers.9.self_attn.attn.impl.v_cache", + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.9.feed_forward.router", + "language_model.model.layers.9.feed_forward.experts", + "language_model.model.layers.9.feed_forward.experts.moe_op", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.9.feed_forward.shared_expert.down_proj", + "language_model.model.layers.10.self_attn.qkv_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.10.self_attn.attn.impl.softmax", + "language_model.model.layers.10.self_attn.attn.impl.matmul_av", + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.10.self_attn.attn.impl.k_cache", + "language_model.model.layers.10.self_attn.attn.impl.v_cache", + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.10.feed_forward.gate_up_proj", + "language_model.model.layers.10.feed_forward.down_proj", + "language_model.model.layers.11.self_attn.qkv_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.11.self_attn.attn.impl.softmax", + "language_model.model.layers.11.self_attn.attn.impl.matmul_av", + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.11.self_attn.attn.impl.k_cache", + "language_model.model.layers.11.self_attn.attn.impl.v_cache", + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.11.feed_forward.router", + "language_model.model.layers.11.feed_forward.experts", + "language_model.model.layers.11.feed_forward.experts.moe_op", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.11.feed_forward.shared_expert.down_proj", + "language_model.model.layers.12.self_attn.qkv_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.12.self_attn.attn.impl.softmax", + "language_model.model.layers.12.self_attn.attn.impl.matmul_av", + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.12.self_attn.attn.impl.k_cache", + "language_model.model.layers.12.self_attn.attn.impl.v_cache", + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.12.feed_forward.gate_up_proj", + "language_model.model.layers.12.feed_forward.down_proj", + "language_model.model.layers.13.self_attn.qkv_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.13.self_attn.attn.impl.softmax", + "language_model.model.layers.13.self_attn.attn.impl.matmul_av", + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.13.self_attn.attn.impl.k_cache", + "language_model.model.layers.13.self_attn.attn.impl.v_cache", + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.13.feed_forward.router", + "language_model.model.layers.13.feed_forward.experts", + "language_model.model.layers.13.feed_forward.experts.moe_op", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.13.feed_forward.shared_expert.down_proj", + "language_model.model.layers.14.self_attn.qkv_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.14.self_attn.attn.impl.softmax", + "language_model.model.layers.14.self_attn.attn.impl.matmul_av", + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.14.self_attn.attn.impl.k_cache", + "language_model.model.layers.14.self_attn.attn.impl.v_cache", + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.14.feed_forward.gate_up_proj", + "language_model.model.layers.14.feed_forward.down_proj", + "language_model.model.layers.15.self_attn.qkv_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.15.self_attn.attn.impl.softmax", + "language_model.model.layers.15.self_attn.attn.impl.matmul_av", + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.15.self_attn.attn.impl.k_cache", + "language_model.model.layers.15.self_attn.attn.impl.v_cache", + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.15.feed_forward.router", + "language_model.model.layers.15.feed_forward.experts", + "language_model.model.layers.15.feed_forward.experts.moe_op", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.15.feed_forward.shared_expert.down_proj", + "language_model.model.layers.16.self_attn.qkv_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.16.self_attn.attn.impl.softmax", + "language_model.model.layers.16.self_attn.attn.impl.matmul_av", + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.16.self_attn.attn.impl.k_cache", + "language_model.model.layers.16.self_attn.attn.impl.v_cache", + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.16.feed_forward.gate_up_proj", + "language_model.model.layers.16.feed_forward.down_proj", + "language_model.model.layers.17.self_attn.qkv_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.17.self_attn.attn.impl.softmax", + "language_model.model.layers.17.self_attn.attn.impl.matmul_av", + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.17.self_attn.attn.impl.k_cache", + "language_model.model.layers.17.self_attn.attn.impl.v_cache", + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.17.feed_forward.router", + "language_model.model.layers.17.feed_forward.experts", + "language_model.model.layers.17.feed_forward.experts.moe_op", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.17.feed_forward.shared_expert.down_proj", + "language_model.model.layers.18.self_attn.qkv_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.18.self_attn.attn.impl.softmax", + "language_model.model.layers.18.self_attn.attn.impl.matmul_av", + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.18.self_attn.attn.impl.k_cache", + "language_model.model.layers.18.self_attn.attn.impl.v_cache", + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.18.feed_forward.gate_up_proj", + "language_model.model.layers.18.feed_forward.down_proj", + "language_model.model.layers.19.self_attn.qkv_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.19.self_attn.attn.impl.softmax", + "language_model.model.layers.19.self_attn.attn.impl.matmul_av", + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.19.self_attn.attn.impl.k_cache", + "language_model.model.layers.19.self_attn.attn.impl.v_cache", + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.19.feed_forward.router", + "language_model.model.layers.19.feed_forward.experts", + "language_model.model.layers.19.feed_forward.experts.moe_op", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.19.feed_forward.shared_expert.down_proj", + "language_model.model.layers.20.self_attn.qkv_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.20.self_attn.attn.impl.softmax", + "language_model.model.layers.20.self_attn.attn.impl.matmul_av", + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.20.self_attn.attn.impl.k_cache", + "language_model.model.layers.20.self_attn.attn.impl.v_cache", + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.20.feed_forward.gate_up_proj", + "language_model.model.layers.20.feed_forward.down_proj", + "language_model.model.layers.21.self_attn.qkv_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.21.self_attn.attn.impl.softmax", + "language_model.model.layers.21.self_attn.attn.impl.matmul_av", + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.21.self_attn.attn.impl.k_cache", + "language_model.model.layers.21.self_attn.attn.impl.v_cache", + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.21.feed_forward.router", + "language_model.model.layers.21.feed_forward.experts", + "language_model.model.layers.21.feed_forward.experts.moe_op", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.21.feed_forward.shared_expert.down_proj", + "language_model.model.layers.22.self_attn.qkv_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.22.self_attn.attn.impl.softmax", + "language_model.model.layers.22.self_attn.attn.impl.matmul_av", + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.22.self_attn.attn.impl.k_cache", + "language_model.model.layers.22.self_attn.attn.impl.v_cache", + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.22.feed_forward.gate_up_proj", + "language_model.model.layers.22.feed_forward.down_proj", + "language_model.model.layers.23.self_attn.qkv_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.23.self_attn.attn.impl.softmax", + "language_model.model.layers.23.self_attn.attn.impl.matmul_av", + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.23.self_attn.attn.impl.k_cache", + "language_model.model.layers.23.self_attn.attn.impl.v_cache", + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.23.feed_forward.router", + "language_model.model.layers.23.feed_forward.experts", + "language_model.model.layers.23.feed_forward.experts.moe_op", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.23.feed_forward.shared_expert.down_proj", + "language_model.model.layers.24.self_attn.qkv_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.24.self_attn.attn.impl.softmax", + "language_model.model.layers.24.self_attn.attn.impl.matmul_av", + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.24.self_attn.attn.impl.k_cache", + "language_model.model.layers.24.self_attn.attn.impl.v_cache", + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.24.feed_forward.gate_up_proj", + "language_model.model.layers.24.feed_forward.down_proj", + "language_model.model.layers.25.self_attn.qkv_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.25.self_attn.attn.impl.softmax", + "language_model.model.layers.25.self_attn.attn.impl.matmul_av", + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.25.self_attn.attn.impl.k_cache", + "language_model.model.layers.25.self_attn.attn.impl.v_cache", + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.25.feed_forward.router", + "language_model.model.layers.25.feed_forward.experts", + "language_model.model.layers.25.feed_forward.experts.moe_op", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.25.feed_forward.shared_expert.down_proj", + "language_model.model.layers.26.self_attn.qkv_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.26.self_attn.attn.impl.softmax", + "language_model.model.layers.26.self_attn.attn.impl.matmul_av", + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.26.self_attn.attn.impl.k_cache", + "language_model.model.layers.26.self_attn.attn.impl.v_cache", + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.26.feed_forward.gate_up_proj", + "language_model.model.layers.26.feed_forward.down_proj", + "language_model.model.layers.27.self_attn.qkv_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.27.self_attn.attn.impl.softmax", + "language_model.model.layers.27.self_attn.attn.impl.matmul_av", + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.27.self_attn.attn.impl.k_cache", + "language_model.model.layers.27.self_attn.attn.impl.v_cache", + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.27.feed_forward.router", + "language_model.model.layers.27.feed_forward.experts", + "language_model.model.layers.27.feed_forward.experts.moe_op", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.27.feed_forward.shared_expert.down_proj", + "language_model.model.layers.28.self_attn.qkv_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.28.self_attn.attn.impl.softmax", + "language_model.model.layers.28.self_attn.attn.impl.matmul_av", + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.28.self_attn.attn.impl.k_cache", + "language_model.model.layers.28.self_attn.attn.impl.v_cache", + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.28.feed_forward.gate_up_proj", + "language_model.model.layers.28.feed_forward.down_proj", + "language_model.model.layers.29.self_attn.qkv_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.29.self_attn.attn.impl.softmax", + "language_model.model.layers.29.self_attn.attn.impl.matmul_av", + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.29.self_attn.attn.impl.k_cache", + "language_model.model.layers.29.self_attn.attn.impl.v_cache", + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.29.feed_forward.router", + "language_model.model.layers.29.feed_forward.experts", + "language_model.model.layers.29.feed_forward.experts.moe_op", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.29.feed_forward.shared_expert.down_proj", + "language_model.model.layers.30.self_attn.qkv_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.30.self_attn.attn.impl.softmax", + "language_model.model.layers.30.self_attn.attn.impl.matmul_av", + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.30.self_attn.attn.impl.k_cache", + "language_model.model.layers.30.self_attn.attn.impl.v_cache", + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.30.feed_forward.gate_up_proj", + "language_model.model.layers.30.feed_forward.down_proj", + "language_model.model.layers.31.self_attn.qkv_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.31.self_attn.attn.impl.softmax", + "language_model.model.layers.31.self_attn.attn.impl.matmul_av", + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.31.self_attn.attn.impl.k_cache", + "language_model.model.layers.31.self_attn.attn.impl.v_cache", + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.31.feed_forward.router", + "language_model.model.layers.31.feed_forward.experts", + "language_model.model.layers.31.feed_forward.experts.moe_op", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.31.feed_forward.shared_expert.down_proj", + "language_model.model.layers.32.self_attn.qkv_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.32.self_attn.attn.impl.softmax", + "language_model.model.layers.32.self_attn.attn.impl.matmul_av", + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.32.self_attn.attn.impl.k_cache", + "language_model.model.layers.32.self_attn.attn.impl.v_cache", + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.32.feed_forward.gate_up_proj", + "language_model.model.layers.32.feed_forward.down_proj", + "language_model.model.layers.33.self_attn.qkv_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.33.self_attn.attn.impl.softmax", + "language_model.model.layers.33.self_attn.attn.impl.matmul_av", + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.33.self_attn.attn.impl.k_cache", + "language_model.model.layers.33.self_attn.attn.impl.v_cache", + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.33.feed_forward.router", + "language_model.model.layers.33.feed_forward.experts", + "language_model.model.layers.33.feed_forward.experts.moe_op", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.33.feed_forward.shared_expert.down_proj", + "language_model.model.layers.34.self_attn.qkv_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.34.self_attn.attn.impl.softmax", + "language_model.model.layers.34.self_attn.attn.impl.matmul_av", + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.34.self_attn.attn.impl.k_cache", + "language_model.model.layers.34.self_attn.attn.impl.v_cache", + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.34.feed_forward.gate_up_proj", + "language_model.model.layers.34.feed_forward.down_proj", + "language_model.model.layers.35.self_attn.qkv_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.35.self_attn.attn.impl.softmax", + "language_model.model.layers.35.self_attn.attn.impl.matmul_av", + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.35.self_attn.attn.impl.k_cache", + "language_model.model.layers.35.self_attn.attn.impl.v_cache", + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.35.feed_forward.router", + "language_model.model.layers.35.feed_forward.experts", + "language_model.model.layers.35.feed_forward.experts.moe_op", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.35.feed_forward.shared_expert.down_proj", + "language_model.model.layers.36.self_attn.qkv_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.36.self_attn.attn.impl.softmax", + "language_model.model.layers.36.self_attn.attn.impl.matmul_av", + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.36.self_attn.attn.impl.k_cache", + "language_model.model.layers.36.self_attn.attn.impl.v_cache", + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.36.feed_forward.gate_up_proj", + "language_model.model.layers.36.feed_forward.down_proj", + "language_model.model.layers.37.self_attn.qkv_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.37.self_attn.attn.impl.softmax", + "language_model.model.layers.37.self_attn.attn.impl.matmul_av", + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.37.self_attn.attn.impl.k_cache", + "language_model.model.layers.37.self_attn.attn.impl.v_cache", + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.37.feed_forward.router", + "language_model.model.layers.37.feed_forward.experts", + "language_model.model.layers.37.feed_forward.experts.moe_op", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.37.feed_forward.shared_expert.down_proj", + "language_model.model.layers.38.self_attn.qkv_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.38.self_attn.attn.impl.softmax", + "language_model.model.layers.38.self_attn.attn.impl.matmul_av", + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.38.self_attn.attn.impl.k_cache", + "language_model.model.layers.38.self_attn.attn.impl.v_cache", + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.38.feed_forward.gate_up_proj", + "language_model.model.layers.38.feed_forward.down_proj", + "language_model.model.layers.39.self_attn.qkv_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.39.self_attn.attn.impl.softmax", + "language_model.model.layers.39.self_attn.attn.impl.matmul_av", + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.39.self_attn.attn.impl.k_cache", + "language_model.model.layers.39.self_attn.attn.impl.v_cache", + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.39.feed_forward.router", + "language_model.model.layers.39.feed_forward.experts", + "language_model.model.layers.39.feed_forward.experts.moe_op", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.39.feed_forward.shared_expert.down_proj", + "language_model.model.layers.40.self_attn.qkv_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.40.self_attn.attn.impl.softmax", + "language_model.model.layers.40.self_attn.attn.impl.matmul_av", + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.40.self_attn.attn.impl.k_cache", + "language_model.model.layers.40.self_attn.attn.impl.v_cache", + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.40.feed_forward.gate_up_proj", + "language_model.model.layers.40.feed_forward.down_proj", + "language_model.model.layers.41.self_attn.qkv_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.41.self_attn.attn.impl.softmax", + "language_model.model.layers.41.self_attn.attn.impl.matmul_av", + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.41.self_attn.attn.impl.k_cache", + "language_model.model.layers.41.self_attn.attn.impl.v_cache", + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.41.feed_forward.router", + "language_model.model.layers.41.feed_forward.experts", + "language_model.model.layers.41.feed_forward.experts.moe_op", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.41.feed_forward.shared_expert.down_proj", + "language_model.model.layers.42.self_attn.qkv_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.42.self_attn.attn.impl.softmax", + "language_model.model.layers.42.self_attn.attn.impl.matmul_av", + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.42.self_attn.attn.impl.k_cache", + "language_model.model.layers.42.self_attn.attn.impl.v_cache", + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.42.feed_forward.gate_up_proj", + "language_model.model.layers.42.feed_forward.down_proj", + "language_model.model.layers.43.self_attn.qkv_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.43.self_attn.attn.impl.softmax", + "language_model.model.layers.43.self_attn.attn.impl.matmul_av", + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.43.self_attn.attn.impl.k_cache", + "language_model.model.layers.43.self_attn.attn.impl.v_cache", + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.43.feed_forward.router", + "language_model.model.layers.43.feed_forward.experts", + "language_model.model.layers.43.feed_forward.experts.moe_op", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.43.feed_forward.shared_expert.down_proj", + "language_model.model.layers.44.self_attn.qkv_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.44.self_attn.attn.impl.softmax", + "language_model.model.layers.44.self_attn.attn.impl.matmul_av", + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.44.self_attn.attn.impl.k_cache", + "language_model.model.layers.44.self_attn.attn.impl.v_cache", + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.44.feed_forward.gate_up_proj", + "language_model.model.layers.44.feed_forward.down_proj", + "language_model.model.layers.45.self_attn.qkv_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.45.self_attn.attn.impl.softmax", + "language_model.model.layers.45.self_attn.attn.impl.matmul_av", + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.45.self_attn.attn.impl.k_cache", + "language_model.model.layers.45.self_attn.attn.impl.v_cache", + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.45.feed_forward.router", + "language_model.model.layers.45.feed_forward.experts", + "language_model.model.layers.45.feed_forward.experts.moe_op", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.45.feed_forward.shared_expert.down_proj", + "language_model.model.layers.46.self_attn.qkv_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.46.self_attn.attn.impl.softmax", + "language_model.model.layers.46.self_attn.attn.impl.matmul_av", + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.46.self_attn.attn.impl.k_cache", + "language_model.model.layers.46.self_attn.attn.impl.v_cache", + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.46.feed_forward.gate_up_proj", + "language_model.model.layers.46.feed_forward.down_proj", + "language_model.model.layers.47.self_attn.qkv_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.47.self_attn.attn.impl.softmax", + "language_model.model.layers.47.self_attn.attn.impl.matmul_av", + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.47.self_attn.attn.impl.k_cache", + "language_model.model.layers.47.self_attn.attn.impl.v_cache", + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.47.feed_forward.router", + "language_model.model.layers.47.feed_forward.experts", + "language_model.model.layers.47.feed_forward.experts.moe_op", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.47.feed_forward.shared_expert.down_proj", + "language_model.lm_head" +] \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_7_8.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_7_8.json new file mode 100644 index 000000000000..ab599da002a4 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_7_8.json @@ -0,0 +1,24263 @@ +{ + "GlobalRank": null, + "LocalRank": 7, + "Mode": "DynamicRange", + "Nodes": { + "language_model.model.layers.0.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 11.1875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.373046875 + ] + ] + } + }, + "language_model.model.layers.0.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.95703125 + ] + ] + ], + "outputs": [ + [ + [ + 0.83203125 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.6875 + ] + ], + [ + [ + 6.75 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.96484375 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.6875 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.9375 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 6.75 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.96484375 + ] + ] + ] + }, + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 8.375 + ] + ], + [ + [ + 6.75 + ] + ], + [ + [ + 0.96484375 + ] + ] + ], + "outputs": [ + [ + [ + 0.95703125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.0.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 2.546875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.0.feed_forward.down_proj": { + "inputs": [ + [ + [ + 3.59375 + ] + ] + ], + "outputs": [ + [ + [ + 26.75 + ] + ], + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.66015625 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 10.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.171875 + ] + ] + ], + "outputs": [ + [ + [ + 0.67578125 + ] + ], + [ + [ + 1.3984375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.62109375 + ] + ], + [ + [ + 8.75 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.5 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.62109375 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1484375 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.5 + ] + ] + ] + }, + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.46875 + ] + ], + [ + [ + 8.75 + ] + ], + [ + [ + 1.5 + ] + ] + ], + "outputs": [ + [ + [ + 1.171875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.1.feed_forward.router": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "outputs": [ + [ + [ + 1.984375 + ] + ], + [ + [ + 28.375 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 24.875 + ] + ], + [ + [ + 17.625 + ] + ], + [ + [ + 20.625 + ] + ], + [ + [ + 11.875 + ] + ], + [ + [ + 20.0 + ] + ], + [ + [ + 15.9375 + ] + ], + [ + [ + 12.5 + ] + ], + [ + [ + 16.875 + ] + ], + [ + [ + 26.25 + ] + ], + [ + [ + 11.75 + ] + ], + [ + [ + 17.75 + ] + ], + [ + [ + 14.8125 + ] + ], + [ + [ + 17.75 + ] + ], + [ + [ + 17.25 + ] + ] + ] + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1796875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0849609375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.373046875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.330078125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 1.890625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.1.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 9.375 + ] + ] + ], + "outputs": [ + [ + [ + 1.28125 + ] + ], + [ + [ + 2.2310650564016837e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.43359375 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 9.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2490234375 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.62109375 + ] + ] + ], + "outputs": [ + [ + [ + 0.7734375 + ] + ], + [ + [ + 1.4921875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.5390625 + ] + ], + [ + [ + 6.5625 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.125 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.5390625 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.59765625 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 6.5625 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.125 + ] + ] + ] + }, + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.0625 + ] + ], + [ + [ + 6.5625 + ] + ], + [ + [ + 1.125 + ] + ] + ], + "outputs": [ + [ + [ + 0.62109375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.2.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 6.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.359375 + ] + ] + } + }, + "language_model.model.layers.2.feed_forward.down_proj": { + "inputs": [ + [ + [ + 5.40625 + ] + ] + ], + "outputs": [ + [ + [ + 4.75 + ] + ], + [ + [ + 25.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5234375 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 7.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.72265625 + ] + ] + ], + "outputs": [ + [ + [ + 0.77734375 + ] + ], + [ + [ + 2.40625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.453125 + ] + ] + } + }, + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.08984375 + ] + ], + [ + [ + 5.0 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.72265625 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.08984375 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.72265625 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 5.0 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 0.72265625 + ] + ] + ] + }, + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 1.8203125 + ] + ], + [ + [ + 5.0 + ] + ], + [ + [ + 0.72265625 + ] + ] + ], + "outputs": [ + [ + [ + 0.72265625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.3.feed_forward.router": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.52734375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "outputs": [ + [ + [ + 1.9609375 + ] + ], + [ + [ + 27.625 + ] + ], + [ + [ + 16.875 + ] + ], + [ + [ + 7.8125 + ] + ], + [ + [ + 11.3125 + ] + ], + [ + [ + 25.125 + ] + ], + [ + [ + 16.125 + ] + ], + [ + [ + 29.875 + ] + ], + [ + [ + 4.1875 + ] + ], + [ + [ + 21.25 + ] + ], + [ + [ + 13.75 + ] + ], + [ + [ + 26.875 + ] + ], + [ + [ + 20.5 + ] + ], + [ + [ + 4.625 + ] + ], + [ + [ + 20.375 + ] + ], + [ + [ + 16.75 + ] + ], + [ + [ + 23.625 + ] + ] + ] + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11376953125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1005859375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10498046875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11376953125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11279296875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.38671875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1044921875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11572265625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11865234375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10693359375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 8.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.3.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 5.78125 + ] + ] + ], + "outputs": [ + [ + [ + 2.40625 + ] + ], + [ + [ + 2.0535939723697316e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6484375 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 14.3125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.703125 + ] + ] + ], + "outputs": [ + [ + [ + 1.46875 + ] + ], + [ + [ + 2.328125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.80078125 + ] + ], + [ + [ + 15.0 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.0625 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.80078125 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.59375 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.0 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.0625 + ] + ] + ] + }, + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.5 + ] + ], + [ + [ + 14.9375 + ] + ], + [ + [ + 2.0625 + ] + ] + ], + "outputs": [ + [ + [ + 1.703125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.4.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 6.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.4.feed_forward.down_proj": { + "inputs": [ + [ + [ + 7.59375 + ] + ] + ], + "outputs": [ + [ + [ + 2.796875 + ] + ], + [ + [ + 5.1875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.373046875 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 18.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.5703125 + ] + ] + ], + "outputs": [ + [ + [ + 1.4921875 + ] + ], + [ + [ + 2.40625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.84375 + ] + ], + [ + [ + 14.5625 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.0625 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.84375 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.15625 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.5625 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.0625 + ] + ] + ] + }, + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.375 + ] + ], + [ + [ + 14.5625 + ] + ], + [ + [ + 2.0625 + ] + ] + ], + "outputs": [ + [ + [ + 1.5703125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.5.feed_forward.router": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 2.796875 + ] + ], + [ + [ + 22.375 + ] + ], + [ + [ + 27.875 + ] + ], + [ + [ + 38.5 + ] + ], + [ + [ + 24.5 + ] + ], + [ + [ + 27.875 + ] + ], + [ + [ + 23.25 + ] + ], + [ + [ + 16.0 + ] + ], + [ + [ + 17.625 + ] + ], + [ + [ + 24.0 + ] + ], + [ + [ + 16.875 + ] + ], + [ + [ + 23.25 + ] + ], + [ + [ + 30.125 + ] + ], + [ + [ + 35.25 + ] + ], + [ + [ + 19.25 + ] + ], + [ + [ + 30.0 + ] + ], + [ + [ + 29.875 + ] + ] + ] + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1044921875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.390625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.115234375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11474609375 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.5.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 6.09375 + ] + ] + ], + "outputs": [ + [ + [ + 0.4765625 + ] + ], + [ + [ + 2.0535939723697316e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 14.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.o_proj": { + "inputs": [ + [ + [ + 1.46875 + ] + ] + ], + "outputs": [ + [ + [ + 0.94140625 + ] + ], + [ + [ + 1.96875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.796875 + ] + ], + [ + [ + 11.625 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.421875 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.796875 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4453125 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 11.625 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.421875 + ] + ] + ] + }, + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.5 + ] + ], + [ + [ + 11.625 + ] + ], + [ + [ + 2.421875 + ] + ] + ], + "outputs": [ + [ + [ + 1.46875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.6.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 7.71875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.6.feed_forward.down_proj": { + "inputs": [ + [ + [ + 8.1875 + ] + ] + ], + "outputs": [ + [ + [ + 2.25 + ] + ], + [ + [ + 11.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.412109375 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 7.59375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.o_proj": { + "inputs": [ + [ + [ + 0.546875 + ] + ] + ], + "outputs": [ + [ + [ + 2.1875 + ] + ], + [ + [ + 2.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.41796875 + ] + ], + [ + [ + 5.53125 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.28125 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.41796875 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.80859375 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 5.53125 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 1.28125 + ] + ] + ] + }, + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 4.84375 + ] + ], + [ + [ + 5.53125 + ] + ], + [ + [ + 1.25 + ] + ] + ], + "outputs": [ + [ + [ + 0.546875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.7.feed_forward.router": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "outputs": [ + [ + [ + 3.203125 + ] + ], + [ + [ + 17.375 + ] + ], + [ + [ + 54.5 + ] + ], + [ + [ + 40.0 + ] + ], + [ + [ + 21.375 + ] + ], + [ + [ + 22.25 + ] + ], + [ + [ + 17.375 + ] + ], + [ + [ + 47.25 + ] + ], + [ + [ + 47.5 + ] + ], + [ + [ + 41.0 + ] + ], + [ + [ + 31.875 + ] + ], + [ + [ + 43.5 + ] + ], + [ + [ + 36.25 + ] + ], + [ + [ + 20.75 + ] + ], + [ + [ + 10.875 + ] + ], + [ + [ + 53.5 + ] + ], + [ + [ + 26.75 + ] + ] + ] + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12158203125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10546875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12060546875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12353515625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1083984375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.56640625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08740234375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10546875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08935546875 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 11.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.259765625 + ] + ] + } + }, + "language_model.model.layers.7.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 9.25 + ] + ] + ], + "outputs": [ + [ + [ + 1.265625 + ] + ], + [ + [ + 2.2310650564016837e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 16.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.333984375 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.828125 + ] + ] + ], + "outputs": [ + [ + [ + 2.265625 + ] + ], + [ + [ + 3.390625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.77734375 + ] + ], + [ + [ + 8.875 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.03125 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.77734375 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.78125 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 8.875 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.03125 + ] + ] + ] + }, + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.5 + ] + ], + [ + [ + 8.875 + ] + ], + [ + [ + 4.03125 + ] + ] + ], + "outputs": [ + [ + [ + 2.828125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.8.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 8.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.8.feed_forward.down_proj": { + "inputs": [ + [ + [ + 8.6875 + ] + ] + ], + "outputs": [ + [ + [ + 2.84375 + ] + ], + [ + [ + 9.8125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.703125 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 16.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.8125 + ] + ] + ], + "outputs": [ + [ + [ + 1.609375 + ] + ], + [ + [ + 3.640625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.8046875 + ] + ], + [ + [ + 9.6875 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.25 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.8046875 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.46875 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 9.6875 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.25 + ] + ] + ] + }, + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 9.5625 + ] + ], + [ + [ + 9.6875 + ] + ], + [ + [ + 4.25 + ] + ] + ], + "outputs": [ + [ + [ + 3.8125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.9.feed_forward.router": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "outputs": [ + [ + [ + 5.125 + ] + ], + [ + [ + 34.75 + ] + ], + [ + [ + 63.25 + ] + ], + [ + [ + 48.5 + ] + ], + [ + [ + 44.25 + ] + ], + [ + [ + 34.25 + ] + ], + [ + [ + 22.0 + ] + ], + [ + [ + 74.5 + ] + ], + [ + [ + 24.125 + ] + ], + [ + [ + 57.25 + ] + ], + [ + [ + 21.75 + ] + ], + [ + [ + 68.0 + ] + ], + [ + [ + 46.5 + ] + ], + [ + [ + 73.0 + ] + ], + [ + [ + 22.625 + ] + ], + [ + [ + 38.5 + ] + ], + [ + [ + 34.25 + ] + ] + ] + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1005859375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09326171875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1201171875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.091796875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.103515625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.109375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1064453125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08544921875 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1025390625 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 11.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.9.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 10.625 + ] + ] + ], + "outputs": [ + [ + [ + 0.7109375 + ] + ], + [ + [ + 2.0535939723697316e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 17.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.46875 + ] + ] + ], + "outputs": [ + [ + [ + 1.984375 + ] + ], + [ + [ + 3.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.97265625 + ] + ], + [ + [ + 19.25 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.859375 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.97265625 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.9921875 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.25 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 2.859375 + ] + ] + ] + }, + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.375 + ] + ], + [ + [ + 19.0 + ] + ], + [ + [ + 2.859375 + ] + ] + ], + "outputs": [ + [ + [ + 2.46875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.10.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 9.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.310546875 + ] + ] + } + }, + "language_model.model.layers.10.feed_forward.down_proj": { + "inputs": [ + [ + [ + 13.5625 + ] + ] + ], + "outputs": [ + [ + [ + 3.40625 + ] + ], + [ + [ + 12.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.67578125 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 15.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.546875 + ] + ] + ], + "outputs": [ + [ + [ + 8.625 + ] + ], + [ + [ + 8.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.349609375 + ] + ] + } + }, + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.984375 + ] + ], + [ + [ + 9.625 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.03125 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.984375 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.296875 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 9.625 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.03125 + ] + ] + ] + }, + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.375 + ] + ], + [ + [ + 9.625 + ] + ], + [ + [ + 4.03125 + ] + ] + ], + "outputs": [ + [ + [ + 2.546875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.11.feed_forward.router": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "outputs": [ + [ + [ + 5.25 + ] + ], + [ + [ + 33.25 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 51.5 + ] + ], + [ + [ + 53.25 + ] + ], + [ + [ + 49.0 + ] + ], + [ + [ + 61.25 + ] + ], + [ + [ + 64.0 + ] + ], + [ + [ + 46.25 + ] + ], + [ + [ + 59.75 + ] + ], + [ + [ + 32.0 + ] + ], + [ + [ + 47.0 + ] + ], + [ + [ + 27.625 + ] + ], + [ + [ + 11.9375 + ] + ], + [ + [ + 28.75 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 64.0 + ] + ] + ] + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10986328125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08642578125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.083984375 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08935546875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11181640625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10107421875 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.166015625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1103515625 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 12.9375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.11.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 12.875 + ] + ] + ], + "outputs": [ + [ + [ + 1.6015625 + ] + ], + [ + [ + 1.4070921662533346e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.322265625 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 20.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1259765625 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.84375 + ] + ] + ], + "outputs": [ + [ + [ + 1.796875 + ] + ], + [ + [ + 3.5625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33984375 + ] + ] + } + }, + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.90625 + ] + ], + [ + [ + 17.625 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.828125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.90625 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.84375 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.625 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.828125 + ] + ] + ] + }, + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.125 + ] + ], + [ + [ + 16.25 + ] + ], + [ + [ + 3.40625 + ] + ] + ], + "outputs": [ + [ + [ + 3.359375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.12.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 10.6875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.322265625 + ] + ] + } + }, + "language_model.model.layers.12.feed_forward.down_proj": { + "inputs": [ + [ + [ + 15.5 + ] + ] + ], + "outputs": [ + [ + [ + 9.25 + ] + ], + [ + [ + 20.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.796875 + ] + ] + ], + "outputs": [ + [ + [ + 2.015625 + ] + ], + [ + [ + 6.34375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.93359375 + ] + ], + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.296875 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.93359375 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.796875 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.296875 + ] + ] + ] + }, + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.0 + ] + ], + [ + [ + 17.375 + ] + ], + [ + [ + 3.296875 + ] + ] + ], + "outputs": [ + [ + [ + 2.59375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.13.feed_forward.router": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "outputs": [ + [ + [ + 5.625 + ] + ], + [ + [ + 34.75 + ] + ], + [ + [ + 70.0 + ] + ], + [ + [ + 74.5 + ] + ], + [ + [ + 55.75 + ] + ], + [ + [ + 33.5 + ] + ], + [ + [ + 62.25 + ] + ], + [ + [ + 55.0 + ] + ], + [ + [ + 25.375 + ] + ], + [ + [ + 49.0 + ] + ], + [ + [ + 83.0 + ] + ], + [ + [ + 41.75 + ] + ], + [ + [ + 29.375 + ] + ], + [ + [ + 17.0 + ] + ], + [ + [ + 51.25 + ] + ], + [ + [ + 40.25 + ] + ], + [ + [ + 49.25 + ] + ] + ] + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2275390625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09521484375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10693359375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1005859375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10302734375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 17.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.13.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 18.625 + ] + ] + ], + "outputs": [ + [ + [ + 2.890625 + ] + ], + [ + [ + 2.2310650564016837e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 20.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.o_proj": { + "inputs": [ + [ + [ + 2.65625 + ] + ] + ], + "outputs": [ + [ + [ + 3.546875 + ] + ], + [ + [ + 5.71875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.77734375 + ] + ], + [ + [ + 12.0625 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.53125 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.77734375 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.359375 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 12.0625 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 3.53125 + ] + ] + ] + }, + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.0625 + ] + ], + [ + [ + 12.0625 + ] + ], + [ + [ + 3.53125 + ] + ] + ], + "outputs": [ + [ + [ + 2.65625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.14.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 12.0625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.14.feed_forward.down_proj": { + "inputs": [ + [ + [ + 12.125 + ] + ] + ], + "outputs": [ + [ + [ + 5.375 + ] + ], + [ + [ + 20.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.609375 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 23.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10888671875 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.34375 + ] + ] + ], + "outputs": [ + [ + [ + 20.0 + ] + ], + [ + [ + 21.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.6640625 + ] + ], + [ + [ + 9.9375 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.40625 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.6640625 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.25 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 9.9375 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.40625 + ] + ] + ] + }, + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.875 + ] + ], + [ + [ + 9.9375 + ] + ], + [ + [ + 4.40625 + ] + ] + ], + "outputs": [ + [ + [ + 3.34375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.15.feed_forward.router": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "outputs": [ + [ + [ + 5.875 + ] + ], + [ + [ + 59.25 + ] + ], + [ + [ + 66.5 + ] + ], + [ + [ + 44.5 + ] + ], + [ + [ + 75.0 + ] + ], + [ + [ + 55.25 + ] + ], + [ + [ + 20.375 + ] + ], + [ + [ + 73.5 + ] + ], + [ + [ + 59.0 + ] + ], + [ + [ + 74.0 + ] + ], + [ + [ + 62.75 + ] + ], + [ + [ + 54.25 + ] + ], + [ + [ + 59.75 + ] + ], + [ + [ + 62.5 + ] + ], + [ + [ + 51.75 + ] + ], + [ + [ + 47.25 + ] + ], + [ + [ + 71.5 + ] + ] + ] + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11474609375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09130859375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10693359375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12109375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1025390625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.087890625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 18.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.15.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 29.25 + ] + ] + ], + "outputs": [ + [ + [ + 4.5 + ] + ], + [ + [ + 2.2310650564016837e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.38671875 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 23.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.09375 + ] + ] + ], + "outputs": [ + [ + [ + 2.4375 + ] + ], + [ + [ + 6.3125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.99609375 + ] + ], + [ + [ + 20.0 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.09375 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.99609375 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.71875 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.0 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.09375 + ] + ] + ] + }, + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.875 + ] + ], + [ + [ + 20.0 + ] + ], + [ + [ + 4.09375 + ] + ] + ], + "outputs": [ + [ + [ + 3.09375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.16.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 14.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.16.feed_forward.down_proj": { + "inputs": [ + [ + [ + 21.875 + ] + ] + ], + "outputs": [ + [ + [ + 6.65625 + ] + ], + [ + [ + 27.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.515625 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.o_proj": { + "inputs": [ + [ + [ + 3.171875 + ] + ] + ], + "outputs": [ + [ + [ + 19.5 + ] + ], + [ + [ + 27.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0859375 + ] + ], + [ + [ + 18.125 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.03125 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0859375 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.828125 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.125 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.03125 + ] + ] + ] + }, + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.5 + ] + ], + [ + [ + 18.125 + ] + ], + [ + [ + 4.03125 + ] + ] + ], + "outputs": [ + [ + [ + 3.171875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.17.feed_forward.router": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 7.5 + ] + ], + [ + [ + 83.5 + ] + ], + [ + [ + 61.0 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 36.25 + ] + ], + [ + [ + 101.5 + ] + ], + [ + [ + 80.5 + ] + ], + [ + [ + 51.25 + ] + ], + [ + [ + 80.0 + ] + ], + [ + [ + 51.75 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 45.25 + ] + ], + [ + [ + 50.5 + ] + ], + [ + [ + 88.0 + ] + ], + [ + [ + 42.0 + ] + ], + [ + [ + 115.0 + ] + ], + [ + [ + 64.0 + ] + ] + ] + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.267578125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.46875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2041015625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.140625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1162109375 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0966796875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11572265625 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.17.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 20.875 + ] + ] + ], + "outputs": [ + [ + [ + 15.0 + ] + ], + [ + [ + 2.0535939723697316e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.81640625 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.625 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.28125 + ] + ] + ], + "outputs": [ + [ + [ + 6.625 + ] + ], + [ + [ + 12.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0546875 + ] + ], + [ + [ + 19.625 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.71875 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0546875 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.15625 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.625 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 4.71875 + ] + ] + ] + }, + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.9375 + ] + ], + [ + [ + 19.625 + ] + ], + [ + [ + 4.6875 + ] + ] + ], + "outputs": [ + [ + [ + 4.28125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.18.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 20.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3671875 + ] + ] + } + }, + "language_model.model.layers.18.feed_forward.down_proj": { + "inputs": [ + [ + [ + 17.0 + ] + ] + ], + "outputs": [ + [ + [ + 8.1875 + ] + ], + [ + [ + 354.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5546875 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 44.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.5 + ] + ] + ], + "outputs": [ + [ + [ + 31.5 + ] + ], + [ + [ + 37.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.25 + ] + ], + [ + [ + 16.0 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.0 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.25 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 3.921875 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.0 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.0 + ] + ] + ] + }, + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.375 + ] + ], + [ + [ + 16.0 + ] + ], + [ + [ + 7.0 + ] + ] + ], + "outputs": [ + [ + [ + 6.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.19.feed_forward.router": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "outputs": [ + [ + [ + 6.46875 + ] + ], + [ + [ + 56.0 + ] + ], + [ + [ + 93.0 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 76.5 + ] + ], + [ + [ + 93.0 + ] + ], + [ + [ + 21.375 + ] + ], + [ + [ + 77.5 + ] + ], + [ + [ + 77.0 + ] + ], + [ + [ + 62.75 + ] + ], + [ + [ + 17.0 + ] + ], + [ + [ + 46.5 + ] + ], + [ + [ + 61.0 + ] + ], + [ + [ + 94.0 + ] + ], + [ + [ + 41.0 + ] + ], + [ + [ + 41.75 + ] + ], + [ + [ + 63.75 + ] + ] + ] + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2353515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.36328125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.330078125 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6015625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1240234375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.333984375 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1025390625 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 22.875 + ] + ] + ], + "params": { + "weight": [ + [ + 0.404296875 + ] + ] + } + }, + "language_model.model.layers.19.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 59.75 + ] + ] + ], + "outputs": [ + [ + [ + 61.75 + ] + ], + [ + [ + 2.0535939723697316e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 1.015625 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 53.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.59375 + ] + ] + ], + "outputs": [ + [ + [ + 12.8125 + ] + ], + [ + [ + 22.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1328125 + ] + ], + [ + [ + 14.0625 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.625 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1328125 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.9375 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.0625 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 8.625 + ] + ] + ] + }, + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.75 + ] + ], + [ + [ + 14.0625 + ] + ], + [ + [ + 8.625 + ] + ] + ], + "outputs": [ + [ + [ + 5.59375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.20.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 42.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28515625 + ] + ] + } + }, + "language_model.model.layers.20.feed_forward.down_proj": { + "inputs": [ + [ + [ + 26.625 + ] + ] + ], + "outputs": [ + [ + [ + 10.5 + ] + ], + [ + [ + 41.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.609375 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 58.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.375 + ] + ] + ], + "outputs": [ + [ + [ + 26.0 + ] + ], + [ + [ + 35.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2109375 + ] + ], + [ + [ + 20.25 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.90625 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2109375 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.6875 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.25 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.90625 + ] + ] + ] + }, + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.8125 + ] + ], + [ + [ + 20.25 + ] + ], + [ + [ + 6.90625 + ] + ] + ], + "outputs": [ + [ + [ + 5.375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.21.feed_forward.router": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.45703125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "outputs": [ + [ + [ + 41.75 + ] + ], + [ + [ + 86.5 + ] + ], + [ + [ + 43.0 + ] + ], + [ + [ + 127.5 + ] + ], + [ + [ + 87.5 + ] + ], + [ + [ + 50.5 + ] + ], + [ + [ + 55.0 + ] + ], + [ + [ + 74.0 + ] + ], + [ + [ + 63.75 + ] + ], + [ + [ + 98.5 + ] + ], + [ + [ + 103.5 + ] + ], + [ + [ + 30.5 + ] + ], + [ + [ + 62.5 + ] + ], + [ + [ + 112.0 + ] + ], + [ + [ + 51.75 + ] + ], + [ + [ + 48.0 + ] + ], + [ + [ + 71.0 + ] + ] + ] + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23828125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.43359375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.328125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.234375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2373046875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28515625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11474609375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.80859375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 24.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.31640625 + ] + ] + } + }, + "language_model.model.layers.21.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 28.5 + ] + ] + ], + "outputs": [ + [ + [ + 10.625 + ] + ], + [ + [ + 2.0535939723697316e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.80078125 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 57.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.o_proj": { + "inputs": [ + [ + [ + 4.96875 + ] + ] + ], + "outputs": [ + [ + [ + 15.75 + ] + ], + [ + [ + 28.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0546875 + ] + ], + [ + [ + 19.625 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.65625 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0546875 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.15625 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.625 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 5.65625 + ] + ] + ] + }, + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.6875 + ] + ], + [ + [ + 19.625 + ] + ], + [ + [ + 5.65625 + ] + ] + ], + "outputs": [ + [ + [ + 4.96875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.22.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 46.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.404296875 + ] + ] + } + }, + "language_model.model.layers.22.feed_forward.down_proj": { + "inputs": [ + [ + [ + 22.625 + ] + ] + ], + "outputs": [ + [ + [ + 14.875 + ] + ], + [ + [ + 77.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7109375 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 69.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.203125 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.0625 + ] + ] + ], + "outputs": [ + [ + [ + 70.0 + ] + ], + [ + [ + 79.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.96484375 + ] + ], + [ + [ + 16.125 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.5 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.96484375 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.625 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.125 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 14.5 + ] + ] + ] + }, + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.25 + ] + ], + [ + [ + 15.8125 + ] + ], + [ + [ + 14.5 + ] + ] + ], + "outputs": [ + [ + [ + 7.0625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.23.feed_forward.router": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.474609375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "outputs": [ + [ + [ + 8.125 + ] + ], + [ + [ + 97.0 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 58.5 + ] + ], + [ + [ + 55.25 + ] + ], + [ + [ + 43.75 + ] + ], + [ + [ + 105.0 + ] + ], + [ + [ + 32.25 + ] + ], + [ + [ + 49.5 + ] + ], + [ + [ + 40.25 + ] + ], + [ + [ + 72.0 + ] + ], + [ + [ + 53.25 + ] + ], + [ + [ + 51.25 + ] + ], + [ + [ + 83.0 + ] + ], + [ + [ + 47.0 + ] + ], + [ + [ + 37.5 + ] + ], + [ + [ + 40.5 + ] + ] + ] + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.244140625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.36328125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.373046875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1728515625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1982421875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 23.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.515625 + ] + ] + } + }, + "language_model.model.layers.23.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 27.25 + ] + ] + ], + "outputs": [ + [ + [ + 20.625 + ] + ], + [ + [ + 2.0535939723697316e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.71875 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.o_proj": { + "inputs": [ + [ + [ + 10.75 + ] + ] + ], + "outputs": [ + [ + [ + 48.0 + ] + ], + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.4375 + ] + ], + [ + [ + 24.5 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.875 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4375 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.3125 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 24.5 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 14.875 + ] + ] + ] + }, + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 18.0 + ] + ], + [ + [ + 24.5 + ] + ], + [ + [ + 14.875 + ] + ] + ], + "outputs": [ + [ + [ + 10.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.24.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4296875 + ] + ] + } + }, + "language_model.model.layers.24.feed_forward.down_proj": { + "inputs": [ + [ + [ + 51.25 + ] + ] + ], + "outputs": [ + [ + [ + 21.75 + ] + ], + [ + [ + 100.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8203125 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.59375 + ] + ] + ], + "outputs": [ + [ + [ + 53.0 + ] + ], + [ + [ + 69.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.4140625 + ] + ], + [ + [ + 20.75 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.6875 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4140625 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.0625 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.75 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.6875 + ] + ] + ] + }, + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.25 + ] + ], + [ + [ + 20.75 + ] + ], + [ + [ + 7.6875 + ] + ] + ], + "outputs": [ + [ + [ + 7.59375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.25.feed_forward.router": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.412109375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "outputs": [ + [ + [ + 10.0 + ] + ], + [ + [ + 80.5 + ] + ], + [ + [ + 39.0 + ] + ], + [ + [ + 39.75 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 77.5 + ] + ], + [ + [ + 72.0 + ] + ], + [ + [ + 119.5 + ] + ], + [ + [ + 100.0 + ] + ], + [ + [ + 47.25 + ] + ], + [ + [ + 81.0 + ] + ], + [ + [ + 49.0 + ] + ], + [ + [ + 60.5 + ] + ], + [ + [ + 60.0 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 28.25 + ] + ], + [ + [ + 21.75 + ] + ] + ] + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.287109375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.46875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1630859375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 30.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4140625 + ] + ] + } + }, + "language_model.model.layers.25.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 236.0 + ] + ] + ], + "outputs": [ + [ + [ + 137.0 + ] + ], + [ + [ + 2.0535939723697316e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.73046875 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 63.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.0 + ] + ] + ], + "outputs": [ + [ + [ + 30.875 + ] + ], + [ + [ + 48.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2109375 + ] + ], + [ + [ + 18.375 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.125 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2109375 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.21875 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.375 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.125 + ] + ] + ] + }, + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.5625 + ] + ], + [ + [ + 18.375 + ] + ], + [ + [ + 9.125 + ] + ] + ], + "outputs": [ + [ + [ + 7.0 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.26.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 45.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.44140625 + ] + ] + } + }, + "language_model.model.layers.26.feed_forward.down_proj": { + "inputs": [ + [ + [ + 50.25 + ] + ] + ], + "outputs": [ + [ + [ + 22.125 + ] + ], + [ + [ + 113.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.765625 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.875 + ] + ] + ], + "outputs": [ + [ + [ + 60.25 + ] + ], + [ + [ + 71.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19140625 + ] + ] + } + }, + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1640625 + ] + ], + [ + [ + 17.375 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.75 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1640625 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.3125 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.375 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 12.75 + ] + ] + ] + }, + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.9375 + ] + ], + [ + [ + 17.375 + ] + ], + [ + [ + 12.75 + ] + ] + ], + "outputs": [ + [ + [ + 6.875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.27.feed_forward.router": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.353515625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "outputs": [ + [ + [ + 13.0625 + ] + ], + [ + [ + 47.25 + ] + ], + [ + [ + 36.25 + ] + ], + [ + [ + 52.5 + ] + ], + [ + [ + 30.625 + ] + ], + [ + [ + 61.5 + ] + ], + [ + [ + 61.0 + ] + ], + [ + [ + 52.75 + ] + ], + [ + [ + 127.0 + ] + ], + [ + [ + 77.0 + ] + ], + [ + [ + 170.0 + ] + ], + [ + [ + 78.5 + ] + ], + [ + [ + 98.0 + ] + ], + [ + [ + 23.375 + ] + ], + [ + [ + 52.75 + ] + ], + [ + [ + 40.0 + ] + ], + [ + [ + 27.25 + ] + ] + ] + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.150390625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.306640625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.13671875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.263671875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1318359375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1826171875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10791015625 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 30.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4453125 + ] + ] + } + }, + "language_model.model.layers.27.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 35.75 + ] + ] + ], + "outputs": [ + [ + [ + 11.1875 + ] + ], + [ + [ + 2.0535939723697316e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.55859375 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 78.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.5 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 97.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1640625 + ] + ], + [ + [ + 18.75 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.6875 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1640625 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.1875 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 18.75 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 14.6875 + ] + ] + ] + }, + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 16.25 + ] + ], + [ + [ + 18.75 + ] + ], + [ + [ + 14.625 + ] + ] + ], + "outputs": [ + [ + [ + 6.5 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.28.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.41015625 + ] + ] + } + }, + "language_model.model.layers.28.feed_forward.down_proj": { + "inputs": [ + [ + [ + 79.0 + ] + ] + ], + "outputs": [ + [ + [ + 140.0 + ] + ], + [ + [ + 149.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8203125 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 67.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.0 + ] + ] + ], + "outputs": [ + [ + [ + 78.0 + ] + ], + [ + [ + 84.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.25 + ] + ], + [ + [ + 17.75 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.65625 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.25 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.59375 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.75 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.65625 + ] + ] + ] + }, + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.625 + ] + ], + [ + [ + 17.75 + ] + ], + [ + [ + 7.65625 + ] + ] + ], + "outputs": [ + [ + [ + 5.0 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.29.feed_forward.router": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.333984375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "outputs": [ + [ + [ + 12.4375 + ] + ], + [ + [ + 88.0 + ] + ], + [ + [ + 17.75 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 64.5 + ] + ], + [ + [ + 88.5 + ] + ], + [ + [ + 102.5 + ] + ], + [ + [ + 87.0 + ] + ], + [ + [ + 81.5 + ] + ], + [ + [ + 53.25 + ] + ], + [ + [ + 34.5 + ] + ], + [ + [ + 65.5 + ] + ], + [ + [ + 83.5 + ] + ], + [ + [ + 21.375 + ] + ], + [ + [ + 59.0 + ] + ], + [ + [ + 121.0 + ] + ], + [ + [ + 55.0 + ] + ] + ] + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1806640625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2109375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5078125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15234375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2265625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10498046875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2255859375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12451171875 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 31.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.29.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 71.5 + ] + ] + ], + "outputs": [ + [ + [ + 30.75 + ] + ], + [ + [ + 2.0535939723697316e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.74609375 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.o_proj": { + "inputs": [ + [ + [ + 9.625 + ] + ] + ], + "outputs": [ + [ + [ + 50.25 + ] + ], + [ + [ + 56.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.80859375 + ] + ] + } + }, + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.25 + ] + ], + [ + [ + 17.25 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.6875 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.25 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.3125 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 17.25 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.6875 + ] + ] + ] + }, + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.1875 + ] + ], + [ + [ + 17.25 + ] + ], + [ + [ + 11.6875 + ] + ] + ], + "outputs": [ + [ + [ + 9.625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.30.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 32.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.318359375 + ] + ] + } + }, + "language_model.model.layers.30.feed_forward.down_proj": { + "inputs": [ + [ + [ + 38.5 + ] + ] + ], + "outputs": [ + [ + [ + 30.25 + ] + ], + [ + [ + 159.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.70703125 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 72.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11572265625 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.o_proj": { + "inputs": [ + [ + [ + 11.75 + ] + ] + ], + "outputs": [ + [ + [ + 77.0 + ] + ], + [ + [ + 115.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.46484375 + ] + ] + } + }, + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1015625 + ] + ], + [ + [ + 9.375 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.125 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1015625 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.0 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 9.375 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.125 + ] + ] + ] + }, + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 12.3125 + ] + ], + [ + [ + 9.375 + ] + ], + [ + [ + 13.125 + ] + ] + ], + "outputs": [ + [ + [ + 11.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.31.feed_forward.router": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.326171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "outputs": [ + [ + [ + 12.5625 + ] + ], + [ + [ + 60.75 + ] + ], + [ + [ + 35.25 + ] + ], + [ + [ + 91.5 + ] + ], + [ + [ + 32.25 + ] + ], + [ + [ + 80.5 + ] + ], + [ + [ + 74.0 + ] + ], + [ + [ + 56.25 + ] + ], + [ + [ + 78.5 + ] + ], + [ + [ + 68.0 + ] + ], + [ + [ + 47.5 + ] + ], + [ + [ + 61.5 + ] + ], + [ + [ + 43.25 + ] + ], + [ + [ + 108.0 + ] + ], + [ + [ + 68.5 + ] + ], + [ + [ + 63.0 + ] + ], + [ + [ + 58.0 + ] + ] + ] + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.17578125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.126953125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.181640625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2138671875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1884765625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3515625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11669921875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.228515625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.107421875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2451171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2060546875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26171875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11962890625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.220703125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21875 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 33.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.55078125 + ] + ] + } + }, + "language_model.model.layers.31.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 44.0 + ] + ] + ], + "outputs": [ + [ + [ + 10.9375 + ] + ], + [ + [ + 2.0535939723697316e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.70703125 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 53.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.90625 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 109.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5546875 + ] + ] + } + }, + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.91796875 + ] + ], + [ + [ + 26.875 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.5 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.91796875 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.59375 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 26.875 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 9.5 + ] + ] + ] + }, + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 10.9375 + ] + ], + [ + [ + 26.875 + ] + ], + [ + [ + 9.5 + ] + ] + ], + "outputs": [ + [ + [ + 7.90625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.32.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 32.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.486328125 + ] + ] + } + }, + "language_model.model.layers.32.feed_forward.down_proj": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "outputs": [ + [ + [ + 29.375 + ] + ], + [ + [ + 75.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.65234375 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 44.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1591796875 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.3125 + ] + ] + ], + "outputs": [ + [ + [ + 94.0 + ] + ], + [ + [ + 132.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.609375 + ] + ], + [ + [ + 15.625 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.375 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.609375 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.59375 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.625 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.375 + ] + ] + ] + }, + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 19.75 + ] + ], + [ + [ + 15.625 + ] + ], + [ + [ + 7.375 + ] + ] + ], + "outputs": [ + [ + [ + 5.3125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.33.feed_forward.router": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3359375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "outputs": [ + [ + [ + 14.25 + ] + ], + [ + [ + 65.0 + ] + ], + [ + [ + 79.0 + ] + ], + [ + [ + 97.0 + ] + ], + [ + [ + 69.5 + ] + ], + [ + [ + 46.25 + ] + ], + [ + [ + 75.0 + ] + ], + [ + [ + 81.0 + ] + ], + [ + [ + 99.0 + ] + ], + [ + [ + 40.75 + ] + ], + [ + [ + 84.0 + ] + ], + [ + [ + 58.0 + ] + ], + [ + [ + 91.5 + ] + ], + [ + [ + 67.5 + ] + ], + [ + [ + 151.0 + ] + ], + [ + [ + 39.25 + ] + ], + [ + [ + 53.0 + ] + ] + ] + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.29296875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1181640625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11376953125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09619140625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.236328125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.25390625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2216796875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.12890625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11181640625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09228515625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.232421875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.384765625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09130859375 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 64.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.373046875 + ] + ] + } + }, + "language_model.model.layers.33.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 59.75 + ] + ] + ], + "outputs": [ + [ + [ + 31.125 + ] + ], + [ + [ + 2.0535939723697316e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.66015625 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 54.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.21484375 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.0625 + ] + ] + ], + "outputs": [ + [ + [ + 55.75 + ] + ], + [ + [ + 98.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.390625 + ] + ], + [ + [ + 14.0 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.9375 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.390625 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.28125 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.0 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 7.9375 + ] + ] + ] + }, + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 14.9375 + ] + ], + [ + [ + 14.0 + ] + ], + [ + [ + 7.9375 + ] + ] + ], + "outputs": [ + [ + [ + 6.0625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.34.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.291015625 + ] + ] + } + }, + "language_model.model.layers.34.feed_forward.down_proj": { + "inputs": [ + [ + [ + 90.0 + ] + ] + ], + "outputs": [ + [ + [ + 36.5 + ] + ], + [ + [ + 162.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.41796875 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 54.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.o_proj": { + "inputs": [ + [ + [ + 6.34375 + ] + ] + ], + "outputs": [ + [ + [ + 110.5 + ] + ], + [ + [ + 116.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.0546875 + ] + ], + [ + [ + 16.125 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.0625 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.0546875 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 5.65625 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.125 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.0625 + ] + ] + ] + }, + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 11.8125 + ] + ], + [ + [ + 16.125 + ] + ], + [ + [ + 11.0625 + ] + ] + ], + "outputs": [ + [ + [ + 6.34375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.35.feed_forward.router": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "outputs": [ + [ + [ + 38.0 + ] + ], + [ + [ + 77.0 + ] + ], + [ + [ + 64.0 + ] + ], + [ + [ + 120.5 + ] + ], + [ + [ + 108.5 + ] + ], + [ + [ + 75.5 + ] + ], + [ + [ + 94.5 + ] + ], + [ + [ + 74.0 + ] + ], + [ + [ + 189.0 + ] + ], + [ + [ + 134.0 + ] + ], + [ + [ + 104.0 + ] + ], + [ + [ + 71.5 + ] + ], + [ + [ + 124.5 + ] + ], + [ + [ + 152.0 + ] + ], + [ + [ + 140.0 + ] + ], + [ + [ + 81.0 + ] + ], + [ + [ + 172.0 + ] + ] + ] + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3203125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09912109375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.193359375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2333984375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09765625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.462890625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1435546875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23046875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.27734375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.265625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0947265625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2490234375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.330078125 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.322265625 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 65.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.35.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 83.5 + ] + ] + ], + "outputs": [ + [ + [ + 16.375 + ] + ], + [ + [ + 2.0535939723697316e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.5546875 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 56.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.o_proj": { + "inputs": [ + [ + [ + 5.9375 + ] + ] + ], + "outputs": [ + [ + [ + 36.75 + ] + ], + [ + [ + 57.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.4921875 + ] + ], + [ + [ + 14.625 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 6.78125 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.4921875 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 4.5625 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 14.625 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 6.78125 + ] + ] + ] + }, + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 18.125 + ] + ], + [ + [ + 14.625 + ] + ], + [ + [ + 6.78125 + ] + ] + ], + "outputs": [ + [ + [ + 5.9375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.36.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 29.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.36.feed_forward.down_proj": { + "inputs": [ + [ + [ + 80.0 + ] + ] + ], + "outputs": [ + [ + [ + 43.0 + ] + ], + [ + [ + 94.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.56640625 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 50.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1396484375 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.o_proj": { + "inputs": [ + [ + [ + 11.1875 + ] + ] + ], + "outputs": [ + [ + [ + 97.5 + ] + ], + [ + [ + 150.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.9921875 + ] + ], + [ + [ + 15.9375 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.9375 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.9921875 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 9.1875 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 15.9375 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 11.9375 + ] + ] + ] + }, + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 22.375 + ] + ], + [ + [ + 15.5625 + ] + ], + [ + [ + 11.9375 + ] + ] + ], + "outputs": [ + [ + [ + 11.1875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.37.feed_forward.router": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.279296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "outputs": [ + [ + [ + 26.5 + ] + ], + [ + [ + 83.0 + ] + ], + [ + [ + 142.0 + ] + ], + [ + [ + 128.0 + ] + ], + [ + [ + 106.5 + ] + ], + [ + [ + 103.0 + ] + ], + [ + [ + 138.0 + ] + ], + [ + [ + 79.0 + ] + ], + [ + [ + 132.0 + ] + ], + [ + [ + 106.0 + ] + ], + [ + [ + 178.0 + ] + ], + [ + [ + 126.0 + ] + ], + [ + [ + 151.0 + ] + ], + [ + [ + 97.0 + ] + ], + [ + [ + 166.0 + ] + ], + [ + [ + 106.5 + ] + ], + [ + [ + 121.0 + ] + ] + ] + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.23828125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2314453125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.216796875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.240234375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.22265625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.275390625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.33203125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1376953125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.296875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0888671875 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.271484375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1142578125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1337890625 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 70.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.46484375 + ] + ] + } + }, + "language_model.model.layers.37.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 102.0 + ] + ] + ], + "outputs": [ + [ + [ + 56.5 + ] + ], + [ + [ + 2.2310650564016837e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 1.046875 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 47.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1455078125 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.o_proj": { + "inputs": [ + [ + [ + 7.78125 + ] + ] + ], + "outputs": [ + [ + [ + 29.75 + ] + ], + [ + [ + 66.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 2.125 + ] + ], + [ + [ + 11.875 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.0 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 2.125 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.21875 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 11.875 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 13.0 + ] + ] + ] + }, + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 25.25 + ] + ], + [ + [ + 11.875 + ] + ], + [ + [ + 13.0 + ] + ] + ], + "outputs": [ + [ + [ + 7.78125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.38.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 31.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.515625 + ] + ] + } + }, + "language_model.model.layers.38.feed_forward.down_proj": { + "inputs": [ + [ + [ + 91.5 + ] + ] + ], + "outputs": [ + [ + [ + 31.125 + ] + ], + [ + [ + 262.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.51171875 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 52.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4296875 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.o_proj": { + "inputs": [ + [ + [ + 11.875 + ] + ] + ], + "outputs": [ + [ + [ + 126.5 + ] + ], + [ + [ + 154.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.8125 + ] + ], + [ + [ + 13.3125 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 20.75 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.8125 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.9375 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 13.3125 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 20.75 + ] + ] + ] + }, + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 20.375 + ] + ], + [ + [ + 13.3125 + ] + ], + [ + [ + 20.5 + ] + ] + ], + "outputs": [ + [ + [ + 11.875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.39.feed_forward.router": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30078125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "outputs": [ + [ + [ + 26.0 + ] + ], + [ + [ + 132.0 + ] + ], + [ + [ + 142.0 + ] + ], + [ + [ + 75.5 + ] + ], + [ + [ + 158.0 + ] + ], + [ + [ + 62.75 + ] + ], + [ + [ + 76.5 + ] + ], + [ + [ + 179.0 + ] + ], + [ + [ + 134.0 + ] + ], + [ + [ + 80.5 + ] + ], + [ + [ + 131.0 + ] + ], + [ + [ + 152.0 + ] + ], + [ + [ + 114.0 + ] + ], + [ + [ + 120.5 + ] + ], + [ + [ + 100.0 + ] + ], + [ + [ + 120.0 + ] + ], + [ + [ + 155.0 + ] + ] + ] + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.154296875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1748046875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11474609375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.19921875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1494140625 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1123046875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.201171875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.314453125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.283203125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1474609375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2421875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.255859375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2001953125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.08935546875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.208984375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1689453125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1484375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 62.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3984375 + ] + ] + } + }, + "language_model.model.layers.39.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 137.0 + ] + ] + ], + "outputs": [ + [ + [ + 18.25 + ] + ], + [ + [ + 2.0535939723697316e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.40625 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 46.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.o_proj": { + "inputs": [ + [ + [ + 12.0625 + ] + ] + ], + "outputs": [ + [ + [ + 94.0 + ] + ], + [ + [ + 102.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.455078125 + ] + ] + } + }, + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.375 + ] + ], + [ + [ + 23.875 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.75 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.375 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 11.5625 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 23.875 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 12.75 + ] + ] + ] + }, + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.6875 + ] + ], + [ + [ + 23.875 + ] + ], + [ + [ + 12.75 + ] + ] + ], + "outputs": [ + [ + [ + 12.0625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.40.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 41.75 + ] + ] + ], + "params": { + "weight": [ + [ + 0.4453125 + ] + ] + } + }, + "language_model.model.layers.40.feed_forward.down_proj": { + "inputs": [ + [ + [ + 191.0 + ] + ] + ], + "outputs": [ + [ + [ + 28.5 + ] + ], + [ + [ + 127.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.609375 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 35.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1611328125 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.o_proj": { + "inputs": [ + [ + [ + 9.375 + ] + ] + ], + "outputs": [ + [ + [ + 160.0 + ] + ], + [ + [ + 177.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.15625 + ] + ], + [ + [ + 19.5 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.375 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.15625 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.625 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.5 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.375 + ] + ] + ] + }, + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.875 + ] + ], + [ + [ + 19.375 + ] + ], + [ + [ + 10.375 + ] + ] + ], + "outputs": [ + [ + [ + 9.375 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.41.feed_forward.router": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "outputs": [ + [ + [ + 48.5 + ] + ], + [ + [ + 204.0 + ] + ], + [ + [ + 163.0 + ] + ], + [ + [ + 103.0 + ] + ], + [ + [ + 178.0 + ] + ], + [ + [ + 149.0 + ] + ], + [ + [ + 249.0 + ] + ], + [ + [ + 147.0 + ] + ], + [ + [ + 189.0 + ] + ], + [ + [ + 165.0 + ] + ], + [ + [ + 132.0 + ] + ], + [ + [ + 213.0 + ] + ], + [ + [ + 105.5 + ] + ], + [ + [ + 161.0 + ] + ], + [ + [ + 179.0 + ] + ], + [ + [ + 137.0 + ] + ], + [ + [ + 160.0 + ] + ] + ] + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1298828125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1357421875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10888671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2890625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1962890625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1787109375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2236328125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.185546875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1650390625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2080078125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1572265625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09814453125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.3125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10498046875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.28125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1513671875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.337890625 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.189453125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 66.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.58203125 + ] + ] + } + }, + "language_model.model.layers.41.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 264.0 + ] + ] + ], + "outputs": [ + [ + [ + 24.625 + ] + ], + [ + [ + 2.2310650564016837e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.53125 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 39.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.o_proj": { + "inputs": [ + [ + [ + 14.3125 + ] + ] + ], + "outputs": [ + [ + [ + 65.0 + ] + ], + [ + [ + 146.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.71484375 + ] + ] + } + }, + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.296875 + ] + ], + [ + [ + 20.0 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 15.125 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.296875 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.75 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.0 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 15.125 + ] + ] + ] + }, + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.875 + ] + ], + [ + [ + 20.0 + ] + ], + [ + [ + 15.125 + ] + ] + ], + "outputs": [ + [ + [ + 14.3125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.42.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 43.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.62890625 + ] + ] + } + }, + "language_model.model.layers.42.feed_forward.down_proj": { + "inputs": [ + [ + [ + 160.0 + ] + ] + ], + "outputs": [ + [ + [ + 70.5 + ] + ], + [ + [ + 218.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.490234375 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 43.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1171875 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.o_proj": { + "inputs": [ + [ + [ + 9.875 + ] + ] + ], + "outputs": [ + [ + [ + 96.0 + ] + ], + [ + [ + 232.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.94921875 + ] + ] + } + }, + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.1796875 + ] + ], + [ + [ + 8.5 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 14.875 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.1796875 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 8.1875 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 8.5 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 14.875 + ] + ] + ] + }, + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 13.5625 + ] + ], + [ + [ + 8.5 + ] + ], + [ + [ + 14.875 + ] + ] + ], + "outputs": [ + [ + [ + 9.875 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.43.feed_forward.router": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2177734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "outputs": [ + [ + [ + 43.0 + ] + ], + [ + [ + 193.0 + ] + ], + [ + [ + 134.0 + ] + ], + [ + [ + 62.25 + ] + ], + [ + [ + 324.0 + ] + ], + [ + [ + 160.0 + ] + ], + [ + [ + 152.0 + ] + ], + [ + [ + 134.0 + ] + ], + [ + [ + 167.0 + ] + ], + [ + [ + 122.0 + ] + ], + [ + [ + 130.0 + ] + ], + [ + [ + 134.0 + ] + ], + [ + [ + 122.5 + ] + ], + [ + [ + 173.0 + ] + ], + [ + [ + 221.0 + ] + ], + [ + [ + 211.0 + ] + ], + [ + [ + 157.0 + ] + ] + ] + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.349609375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09716796875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.134765625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1328125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11767578125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.26953125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2294921875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.177734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.212890625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1416015625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1923828125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0986328125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1904296875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.30859375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1279296875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2021484375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1015625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10400390625 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09423828125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.248046875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2158203125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2099609375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 68.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.298828125 + ] + ] + } + }, + "language_model.model.layers.43.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 219.0 + ] + ] + ], + "outputs": [ + [ + [ + 62.25 + ] + ], + [ + [ + 2.2310650564016837e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.71484375 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 29.375 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1533203125 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.o_proj": { + "inputs": [ + [ + [ + 17.375 + ] + ] + ], + "outputs": [ + [ + [ + 81.0 + ] + ], + [ + [ + 170.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2734375 + ] + ] + } + }, + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.2890625 + ] + ], + [ + [ + 27.0 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 18.875 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.2890625 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 17.5 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 27.0 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 18.875 + ] + ] + ] + }, + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 15.3125 + ] + ], + [ + [ + 27.0 + ] + ], + [ + [ + 18.875 + ] + ] + ], + "outputs": [ + [ + [ + 16.75 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.44.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 42.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.24609375 + ] + ] + } + }, + "language_model.model.layers.44.feed_forward.down_proj": { + "inputs": [ + [ + [ + 338.0 + ] + ] + ], + "outputs": [ + [ + [ + 75.5 + ] + ], + [ + [ + 200.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7109375 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 28.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.o_proj": { + "inputs": [ + [ + [ + 15.0625 + ] + ] + ], + "outputs": [ + [ + [ + 170.0 + ] + ], + [ + [ + 258.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.39453125 + ] + ] + } + }, + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.8359375 + ] + ], + [ + [ + 19.5 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 16.75 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.8359375 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 13.75 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 19.5 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 16.75 + ] + ] + ] + }, + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 22.0 + ] + ], + [ + [ + 19.5 + ] + ], + [ + [ + 16.75 + ] + ] + ], + "outputs": [ + [ + [ + 15.0625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.45.feed_forward.router": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "outputs": [ + [ + [ + 47.25 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 133.0 + ] + ], + [ + [ + 23.5 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 121.0 + ] + ], + [ + [ + 0.0 + ] + ], + [ + [ + 0.0 + ] + ] + ] + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.162109375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.197265625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1845703125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.15625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1708984375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11474609375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.142578125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2412109375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1943359375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.34375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16796875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.302734375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.16015625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.095703125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1767578125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.20703125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.169921875 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.119140625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2578125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.099609375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.09033203125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1220703125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.0927734375 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1953125 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 52.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2119140625 + ] + ] + } + }, + "language_model.model.layers.45.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 207.0 + ] + ] + ], + "outputs": [ + [ + [ + 35.75 + ] + ], + [ + [ + 2.0535939723697316e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.294921875 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 27.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2470703125 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.o_proj": { + "inputs": [ + [ + [ + 15.3125 + ] + ] + ], + "outputs": [ + [ + [ + 126.0 + ] + ], + [ + [ + 221.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.138671875 + ] + ] + } + }, + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 1.609375 + ] + ], + [ + [ + 20.25 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 15.6875 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 1.609375 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 12.8125 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 20.25 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 15.6875 + ] + ] + ] + }, + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 18.875 + ] + ], + [ + [ + 20.25 + ] + ], + [ + [ + 15.6875 + ] + ] + ], + "outputs": [ + [ + [ + 15.3125 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.46.feed_forward.gate_up_proj": { + "inputs": [ + [ + [ + 40.5 + ] + ] + ], + "params": { + "weight": [ + [ + 0.63671875 + ] + ] + } + }, + "language_model.model.layers.46.feed_forward.down_proj": { + "inputs": [ + [ + [ + 2480.0 + ] + ] + ], + "outputs": [ + [ + [ + 1680.0 + ] + ], + [ + [ + 2400.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.51171875 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.qkv_proj": { + "inputs": [ + [ + [ + 22.25 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1669921875 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.o_proj": { + "inputs": [ + [ + [ + 8.875 + ] + ] + ], + "outputs": [ + [ + [ + 96.5 + ] + ], + [ + [ + 125.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7421875 + ] + ] + } + }, + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk": { + "inputs": [ + [ + [ + 0.59375 + ] + ], + [ + [ + 16.75 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.matmul_av": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 10.25 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 0.59375 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul": { + "inputs": [ + [ + [ + 1.0 + ] + ], + [ + [ + 7.40625 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.k_cache": { + "inputs": [ + [ + [ + 16.75 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.v_cache": { + "inputs": [ + [ + [ + 10.25 + ] + ] + ] + }, + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention": { + "inputs": [ + [ + [ + 7.46875 + ] + ], + [ + [ + 16.75 + ] + ], + [ + [ + 10.25 + ] + ] + ], + "outputs": [ + [ + [ + 8.625 + ] + ], + [ + [ + 1.0 + ] + ] + ] + }, + "language_model.model.layers.47.feed_forward.router": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.37890625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "outputs": [ + [ + [ + 1080.0 + ] + ], + [ + [ + 36.5 + ] + ], + [ + [ + 218.0 + ] + ], + [ + [ + 120.5 + ] + ], + [ + [ + 137.0 + ] + ], + [ + [ + 334.0 + ] + ], + [ + [ + 356.0 + ] + ], + [ + [ + 121.0 + ] + ], + [ + [ + 1184.0 + ] + ], + [ + [ + 488.0 + ] + ], + [ + [ + 170.0 + ] + ], + [ + [ + 169.0 + ] + ], + [ + [ + 79.0 + ] + ], + [ + [ + 88.5 + ] + ], + [ + [ + 394.0 + ] + ], + [ + [ + 422.0 + ] + ], + [ + [ + 468.0 + ] + ] + ] + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.146484375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1552734375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10595703125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.14453125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.11328125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.10205078125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2431640625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1640625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.130859375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.173828125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.1865234375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.158203125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.18359375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.171875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.224609375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.796875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.78515625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.458984375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6796875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.2392578125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.76953125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.69921875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7421875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7890625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.75390625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.6640625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.421875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.7734375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8203125 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.8359375 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15": { + "inputs": [ + [ + [ + 0.0 + ] + ] + ], + "params": { + "weight": [ + [ + 0.796875 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj": { + "inputs": [ + [ + [ + 31.125 + ] + ] + ], + "params": { + "weight": [ + [ + 0.56640625 + ] + ] + } + }, + "language_model.model.layers.47.feed_forward.shared_expert.down_proj": { + "inputs": [ + [ + [ + 2288.0 + ] + ] + ], + "outputs": [ + [ + [ + 1112.0 + ] + ], + [ + [ + 2.0535939723697316e+32 + ] + ] + ], + "params": { + "weight": [ + [ + 0.45703125 + ] + ] + } + }, + "language_model.lm_head": { + "inputs": [ + [ + [ + 54.25 + ] + ] + ], + "params": { + "weight": [ + [ + 1.0703125 + ] + ] + } + } + } +} \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_7_8.npz b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_7_8.npz new file mode 100644 index 000000000000..37393071c24e Binary files /dev/null and b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_7_8.npz differ diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_7_8_mod_list.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_7_8_mod_list.json new file mode 100644 index 000000000000..449fce78b77d --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_measure_output_hooks_maxabs_7_8_mod_list.json @@ -0,0 +1,1559 @@ +[ + "vision_model.patch_embedding.linear", + "vision_model.model.layers.0.self_attn.qkv_proj", + "vision_model.model.layers.0.self_attn.o_proj", + "vision_model.model.layers.0.mlp.fc1", + "vision_model.model.layers.0.mlp.fc2", + "vision_model.model.layers.1.self_attn.qkv_proj", + "vision_model.model.layers.1.self_attn.o_proj", + "vision_model.model.layers.1.mlp.fc1", + "vision_model.model.layers.1.mlp.fc2", + "vision_model.model.layers.2.self_attn.qkv_proj", + "vision_model.model.layers.2.self_attn.o_proj", + "vision_model.model.layers.2.mlp.fc1", + "vision_model.model.layers.2.mlp.fc2", + "vision_model.model.layers.3.self_attn.qkv_proj", + "vision_model.model.layers.3.self_attn.o_proj", + "vision_model.model.layers.3.mlp.fc1", + "vision_model.model.layers.3.mlp.fc2", + "vision_model.model.layers.4.self_attn.qkv_proj", + "vision_model.model.layers.4.self_attn.o_proj", + "vision_model.model.layers.4.mlp.fc1", + "vision_model.model.layers.4.mlp.fc2", + "vision_model.model.layers.5.self_attn.qkv_proj", + "vision_model.model.layers.5.self_attn.o_proj", + "vision_model.model.layers.5.mlp.fc1", + "vision_model.model.layers.5.mlp.fc2", + "vision_model.model.layers.6.self_attn.qkv_proj", + "vision_model.model.layers.6.self_attn.o_proj", + "vision_model.model.layers.6.mlp.fc1", + "vision_model.model.layers.6.mlp.fc2", + "vision_model.model.layers.7.self_attn.qkv_proj", + "vision_model.model.layers.7.self_attn.o_proj", + "vision_model.model.layers.7.mlp.fc1", + "vision_model.model.layers.7.mlp.fc2", + "vision_model.model.layers.8.self_attn.qkv_proj", + "vision_model.model.layers.8.self_attn.o_proj", + "vision_model.model.layers.8.mlp.fc1", + "vision_model.model.layers.8.mlp.fc2", + "vision_model.model.layers.9.self_attn.qkv_proj", + "vision_model.model.layers.9.self_attn.o_proj", + "vision_model.model.layers.9.mlp.fc1", + "vision_model.model.layers.9.mlp.fc2", + "vision_model.model.layers.10.self_attn.qkv_proj", + "vision_model.model.layers.10.self_attn.o_proj", + "vision_model.model.layers.10.mlp.fc1", + "vision_model.model.layers.10.mlp.fc2", + "vision_model.model.layers.11.self_attn.qkv_proj", + "vision_model.model.layers.11.self_attn.o_proj", + "vision_model.model.layers.11.mlp.fc1", + "vision_model.model.layers.11.mlp.fc2", + "vision_model.model.layers.12.self_attn.qkv_proj", + "vision_model.model.layers.12.self_attn.o_proj", + "vision_model.model.layers.12.mlp.fc1", + "vision_model.model.layers.12.mlp.fc2", + "vision_model.model.layers.13.self_attn.qkv_proj", + "vision_model.model.layers.13.self_attn.o_proj", + "vision_model.model.layers.13.mlp.fc1", + "vision_model.model.layers.13.mlp.fc2", + "vision_model.model.layers.14.self_attn.qkv_proj", + "vision_model.model.layers.14.self_attn.o_proj", + "vision_model.model.layers.14.mlp.fc1", + "vision_model.model.layers.14.mlp.fc2", + "vision_model.model.layers.15.self_attn.qkv_proj", + "vision_model.model.layers.15.self_attn.o_proj", + "vision_model.model.layers.15.mlp.fc1", + "vision_model.model.layers.15.mlp.fc2", + "vision_model.model.layers.16.self_attn.qkv_proj", + "vision_model.model.layers.16.self_attn.o_proj", + "vision_model.model.layers.16.mlp.fc1", + "vision_model.model.layers.16.mlp.fc2", + "vision_model.model.layers.17.self_attn.qkv_proj", + "vision_model.model.layers.17.self_attn.o_proj", + "vision_model.model.layers.17.mlp.fc1", + "vision_model.model.layers.17.mlp.fc2", + "vision_model.model.layers.18.self_attn.qkv_proj", + "vision_model.model.layers.18.self_attn.o_proj", + "vision_model.model.layers.18.mlp.fc1", + "vision_model.model.layers.18.mlp.fc2", + "vision_model.model.layers.19.self_attn.qkv_proj", + "vision_model.model.layers.19.self_attn.o_proj", + "vision_model.model.layers.19.mlp.fc1", + "vision_model.model.layers.19.mlp.fc2", + "vision_model.model.layers.20.self_attn.qkv_proj", + "vision_model.model.layers.20.self_attn.o_proj", + "vision_model.model.layers.20.mlp.fc1", + "vision_model.model.layers.20.mlp.fc2", + "vision_model.model.layers.21.self_attn.qkv_proj", + "vision_model.model.layers.21.self_attn.o_proj", + "vision_model.model.layers.21.mlp.fc1", + "vision_model.model.layers.21.mlp.fc2", + "vision_model.model.layers.22.self_attn.qkv_proj", + "vision_model.model.layers.22.self_attn.o_proj", + "vision_model.model.layers.22.mlp.fc1", + "vision_model.model.layers.22.mlp.fc2", + "vision_model.model.layers.23.self_attn.qkv_proj", + "vision_model.model.layers.23.self_attn.o_proj", + "vision_model.model.layers.23.mlp.fc1", + "vision_model.model.layers.23.mlp.fc2", + "vision_model.model.layers.24.self_attn.qkv_proj", + "vision_model.model.layers.24.self_attn.o_proj", + "vision_model.model.layers.24.mlp.fc1", + "vision_model.model.layers.24.mlp.fc2", + "vision_model.model.layers.25.self_attn.qkv_proj", + "vision_model.model.layers.25.self_attn.o_proj", + "vision_model.model.layers.25.mlp.fc1", + "vision_model.model.layers.25.mlp.fc2", + "vision_model.model.layers.26.self_attn.qkv_proj", + "vision_model.model.layers.26.self_attn.o_proj", + "vision_model.model.layers.26.mlp.fc1", + "vision_model.model.layers.26.mlp.fc2", + "vision_model.model.layers.27.self_attn.qkv_proj", + "vision_model.model.layers.27.self_attn.o_proj", + "vision_model.model.layers.27.mlp.fc1", + "vision_model.model.layers.27.mlp.fc2", + "vision_model.model.layers.28.self_attn.qkv_proj", + "vision_model.model.layers.28.self_attn.o_proj", + "vision_model.model.layers.28.mlp.fc1", + "vision_model.model.layers.28.mlp.fc2", + "vision_model.model.layers.29.self_attn.qkv_proj", + "vision_model.model.layers.29.self_attn.o_proj", + "vision_model.model.layers.29.mlp.fc1", + "vision_model.model.layers.29.mlp.fc2", + "vision_model.model.layers.30.self_attn.qkv_proj", + "vision_model.model.layers.30.self_attn.o_proj", + "vision_model.model.layers.30.mlp.fc1", + "vision_model.model.layers.30.mlp.fc2", + "vision_model.model.layers.31.self_attn.qkv_proj", + "vision_model.model.layers.31.self_attn.o_proj", + "vision_model.model.layers.31.mlp.fc1", + "vision_model.model.layers.31.mlp.fc2", + "vision_model.model.layers.32.self_attn.qkv_proj", + "vision_model.model.layers.32.self_attn.o_proj", + "vision_model.model.layers.32.mlp.fc1", + "vision_model.model.layers.32.mlp.fc2", + "vision_model.model.layers.33.self_attn.qkv_proj", + "vision_model.model.layers.33.self_attn.o_proj", + "vision_model.model.layers.33.mlp.fc1", + "vision_model.model.layers.33.mlp.fc2", + "vision_model.vision_adapter.mlp.fc1", + "vision_model.vision_adapter.mlp.fc2", + "multi_modal_projector.linear_1", + "language_model.model.layers.0.self_attn.qkv_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.0.self_attn.attn.impl.softmax", + "language_model.model.layers.0.self_attn.attn.impl.matmul_av", + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.0.self_attn.attn.impl.k_cache", + "language_model.model.layers.0.self_attn.attn.impl.v_cache", + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.0.feed_forward.gate_up_proj", + "language_model.model.layers.0.feed_forward.down_proj", + "language_model.model.layers.1.self_attn.qkv_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.1.self_attn.attn.impl.softmax", + "language_model.model.layers.1.self_attn.attn.impl.matmul_av", + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.1.self_attn.attn.impl.k_cache", + "language_model.model.layers.1.self_attn.attn.impl.v_cache", + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.1.feed_forward.router", + "language_model.model.layers.1.feed_forward.experts", + "language_model.model.layers.1.feed_forward.experts.moe_op", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.1.feed_forward.shared_expert.down_proj", + "language_model.model.layers.2.self_attn.qkv_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.2.self_attn.attn.impl.softmax", + "language_model.model.layers.2.self_attn.attn.impl.matmul_av", + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.2.self_attn.attn.impl.k_cache", + "language_model.model.layers.2.self_attn.attn.impl.v_cache", + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.2.feed_forward.gate_up_proj", + "language_model.model.layers.2.feed_forward.down_proj", + "language_model.model.layers.3.self_attn.qkv_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.3.self_attn.attn.impl.softmax", + "language_model.model.layers.3.self_attn.attn.impl.matmul_av", + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.3.self_attn.attn.impl.k_cache", + "language_model.model.layers.3.self_attn.attn.impl.v_cache", + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.3.feed_forward.router", + "language_model.model.layers.3.feed_forward.experts", + "language_model.model.layers.3.feed_forward.experts.moe_op", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.3.feed_forward.shared_expert.down_proj", + "language_model.model.layers.4.self_attn.qkv_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.4.self_attn.attn.impl.softmax", + "language_model.model.layers.4.self_attn.attn.impl.matmul_av", + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.4.self_attn.attn.impl.k_cache", + "language_model.model.layers.4.self_attn.attn.impl.v_cache", + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.4.feed_forward.gate_up_proj", + "language_model.model.layers.4.feed_forward.down_proj", + "language_model.model.layers.5.self_attn.qkv_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.5.self_attn.attn.impl.softmax", + "language_model.model.layers.5.self_attn.attn.impl.matmul_av", + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.5.self_attn.attn.impl.k_cache", + "language_model.model.layers.5.self_attn.attn.impl.v_cache", + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.5.feed_forward.router", + "language_model.model.layers.5.feed_forward.experts", + "language_model.model.layers.5.feed_forward.experts.moe_op", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.5.feed_forward.shared_expert.down_proj", + "language_model.model.layers.6.self_attn.qkv_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.6.self_attn.attn.impl.softmax", + "language_model.model.layers.6.self_attn.attn.impl.matmul_av", + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.6.self_attn.attn.impl.k_cache", + "language_model.model.layers.6.self_attn.attn.impl.v_cache", + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.6.feed_forward.gate_up_proj", + "language_model.model.layers.6.feed_forward.down_proj", + "language_model.model.layers.7.self_attn.qkv_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.7.self_attn.attn.impl.softmax", + "language_model.model.layers.7.self_attn.attn.impl.matmul_av", + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.7.self_attn.attn.impl.k_cache", + "language_model.model.layers.7.self_attn.attn.impl.v_cache", + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.7.feed_forward.router", + "language_model.model.layers.7.feed_forward.experts", + "language_model.model.layers.7.feed_forward.experts.moe_op", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.7.feed_forward.shared_expert.down_proj", + "language_model.model.layers.8.self_attn.qkv_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.8.self_attn.attn.impl.softmax", + "language_model.model.layers.8.self_attn.attn.impl.matmul_av", + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.8.self_attn.attn.impl.k_cache", + "language_model.model.layers.8.self_attn.attn.impl.v_cache", + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.8.feed_forward.gate_up_proj", + "language_model.model.layers.8.feed_forward.down_proj", + "language_model.model.layers.9.self_attn.qkv_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.9.self_attn.attn.impl.softmax", + "language_model.model.layers.9.self_attn.attn.impl.matmul_av", + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.9.self_attn.attn.impl.k_cache", + "language_model.model.layers.9.self_attn.attn.impl.v_cache", + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.9.feed_forward.router", + "language_model.model.layers.9.feed_forward.experts", + "language_model.model.layers.9.feed_forward.experts.moe_op", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.9.feed_forward.shared_expert.down_proj", + "language_model.model.layers.10.self_attn.qkv_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.10.self_attn.attn.impl.softmax", + "language_model.model.layers.10.self_attn.attn.impl.matmul_av", + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.10.self_attn.attn.impl.k_cache", + "language_model.model.layers.10.self_attn.attn.impl.v_cache", + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.10.feed_forward.gate_up_proj", + "language_model.model.layers.10.feed_forward.down_proj", + "language_model.model.layers.11.self_attn.qkv_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.11.self_attn.attn.impl.softmax", + "language_model.model.layers.11.self_attn.attn.impl.matmul_av", + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.11.self_attn.attn.impl.k_cache", + "language_model.model.layers.11.self_attn.attn.impl.v_cache", + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.11.feed_forward.router", + "language_model.model.layers.11.feed_forward.experts", + "language_model.model.layers.11.feed_forward.experts.moe_op", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.11.feed_forward.shared_expert.down_proj", + "language_model.model.layers.12.self_attn.qkv_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.12.self_attn.attn.impl.softmax", + "language_model.model.layers.12.self_attn.attn.impl.matmul_av", + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.12.self_attn.attn.impl.k_cache", + "language_model.model.layers.12.self_attn.attn.impl.v_cache", + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.12.feed_forward.gate_up_proj", + "language_model.model.layers.12.feed_forward.down_proj", + "language_model.model.layers.13.self_attn.qkv_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.13.self_attn.attn.impl.softmax", + "language_model.model.layers.13.self_attn.attn.impl.matmul_av", + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.13.self_attn.attn.impl.k_cache", + "language_model.model.layers.13.self_attn.attn.impl.v_cache", + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.13.feed_forward.router", + "language_model.model.layers.13.feed_forward.experts", + "language_model.model.layers.13.feed_forward.experts.moe_op", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.13.feed_forward.shared_expert.down_proj", + "language_model.model.layers.14.self_attn.qkv_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.14.self_attn.attn.impl.softmax", + "language_model.model.layers.14.self_attn.attn.impl.matmul_av", + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.14.self_attn.attn.impl.k_cache", + "language_model.model.layers.14.self_attn.attn.impl.v_cache", + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.14.feed_forward.gate_up_proj", + "language_model.model.layers.14.feed_forward.down_proj", + "language_model.model.layers.15.self_attn.qkv_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.15.self_attn.attn.impl.softmax", + "language_model.model.layers.15.self_attn.attn.impl.matmul_av", + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.15.self_attn.attn.impl.k_cache", + "language_model.model.layers.15.self_attn.attn.impl.v_cache", + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.15.feed_forward.router", + "language_model.model.layers.15.feed_forward.experts", + "language_model.model.layers.15.feed_forward.experts.moe_op", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.15.feed_forward.shared_expert.down_proj", + "language_model.model.layers.16.self_attn.qkv_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.16.self_attn.attn.impl.softmax", + "language_model.model.layers.16.self_attn.attn.impl.matmul_av", + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.16.self_attn.attn.impl.k_cache", + "language_model.model.layers.16.self_attn.attn.impl.v_cache", + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.16.feed_forward.gate_up_proj", + "language_model.model.layers.16.feed_forward.down_proj", + "language_model.model.layers.17.self_attn.qkv_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.17.self_attn.attn.impl.softmax", + "language_model.model.layers.17.self_attn.attn.impl.matmul_av", + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.17.self_attn.attn.impl.k_cache", + "language_model.model.layers.17.self_attn.attn.impl.v_cache", + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.17.feed_forward.router", + "language_model.model.layers.17.feed_forward.experts", + "language_model.model.layers.17.feed_forward.experts.moe_op", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.17.feed_forward.shared_expert.down_proj", + "language_model.model.layers.18.self_attn.qkv_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.18.self_attn.attn.impl.softmax", + "language_model.model.layers.18.self_attn.attn.impl.matmul_av", + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.18.self_attn.attn.impl.k_cache", + "language_model.model.layers.18.self_attn.attn.impl.v_cache", + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.18.feed_forward.gate_up_proj", + "language_model.model.layers.18.feed_forward.down_proj", + "language_model.model.layers.19.self_attn.qkv_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.19.self_attn.attn.impl.softmax", + "language_model.model.layers.19.self_attn.attn.impl.matmul_av", + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.19.self_attn.attn.impl.k_cache", + "language_model.model.layers.19.self_attn.attn.impl.v_cache", + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.19.feed_forward.router", + "language_model.model.layers.19.feed_forward.experts", + "language_model.model.layers.19.feed_forward.experts.moe_op", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.19.feed_forward.shared_expert.down_proj", + "language_model.model.layers.20.self_attn.qkv_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.20.self_attn.attn.impl.softmax", + "language_model.model.layers.20.self_attn.attn.impl.matmul_av", + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.20.self_attn.attn.impl.k_cache", + "language_model.model.layers.20.self_attn.attn.impl.v_cache", + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.20.feed_forward.gate_up_proj", + "language_model.model.layers.20.feed_forward.down_proj", + "language_model.model.layers.21.self_attn.qkv_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.21.self_attn.attn.impl.softmax", + "language_model.model.layers.21.self_attn.attn.impl.matmul_av", + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.21.self_attn.attn.impl.k_cache", + "language_model.model.layers.21.self_attn.attn.impl.v_cache", + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.21.feed_forward.router", + "language_model.model.layers.21.feed_forward.experts", + "language_model.model.layers.21.feed_forward.experts.moe_op", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.21.feed_forward.shared_expert.down_proj", + "language_model.model.layers.22.self_attn.qkv_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.22.self_attn.attn.impl.softmax", + "language_model.model.layers.22.self_attn.attn.impl.matmul_av", + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.22.self_attn.attn.impl.k_cache", + "language_model.model.layers.22.self_attn.attn.impl.v_cache", + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.22.feed_forward.gate_up_proj", + "language_model.model.layers.22.feed_forward.down_proj", + "language_model.model.layers.23.self_attn.qkv_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.23.self_attn.attn.impl.softmax", + "language_model.model.layers.23.self_attn.attn.impl.matmul_av", + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.23.self_attn.attn.impl.k_cache", + "language_model.model.layers.23.self_attn.attn.impl.v_cache", + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.23.feed_forward.router", + "language_model.model.layers.23.feed_forward.experts", + "language_model.model.layers.23.feed_forward.experts.moe_op", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.23.feed_forward.shared_expert.down_proj", + "language_model.model.layers.24.self_attn.qkv_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.24.self_attn.attn.impl.softmax", + "language_model.model.layers.24.self_attn.attn.impl.matmul_av", + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.24.self_attn.attn.impl.k_cache", + "language_model.model.layers.24.self_attn.attn.impl.v_cache", + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.24.feed_forward.gate_up_proj", + "language_model.model.layers.24.feed_forward.down_proj", + "language_model.model.layers.25.self_attn.qkv_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.25.self_attn.attn.impl.softmax", + "language_model.model.layers.25.self_attn.attn.impl.matmul_av", + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.25.self_attn.attn.impl.k_cache", + "language_model.model.layers.25.self_attn.attn.impl.v_cache", + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.25.feed_forward.router", + "language_model.model.layers.25.feed_forward.experts", + "language_model.model.layers.25.feed_forward.experts.moe_op", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.25.feed_forward.shared_expert.down_proj", + "language_model.model.layers.26.self_attn.qkv_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.26.self_attn.attn.impl.softmax", + "language_model.model.layers.26.self_attn.attn.impl.matmul_av", + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.26.self_attn.attn.impl.k_cache", + "language_model.model.layers.26.self_attn.attn.impl.v_cache", + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.26.feed_forward.gate_up_proj", + "language_model.model.layers.26.feed_forward.down_proj", + "language_model.model.layers.27.self_attn.qkv_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.27.self_attn.attn.impl.softmax", + "language_model.model.layers.27.self_attn.attn.impl.matmul_av", + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.27.self_attn.attn.impl.k_cache", + "language_model.model.layers.27.self_attn.attn.impl.v_cache", + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.27.feed_forward.router", + "language_model.model.layers.27.feed_forward.experts", + "language_model.model.layers.27.feed_forward.experts.moe_op", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.27.feed_forward.shared_expert.down_proj", + "language_model.model.layers.28.self_attn.qkv_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.28.self_attn.attn.impl.softmax", + "language_model.model.layers.28.self_attn.attn.impl.matmul_av", + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.28.self_attn.attn.impl.k_cache", + "language_model.model.layers.28.self_attn.attn.impl.v_cache", + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.28.feed_forward.gate_up_proj", + "language_model.model.layers.28.feed_forward.down_proj", + "language_model.model.layers.29.self_attn.qkv_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.29.self_attn.attn.impl.softmax", + "language_model.model.layers.29.self_attn.attn.impl.matmul_av", + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.29.self_attn.attn.impl.k_cache", + "language_model.model.layers.29.self_attn.attn.impl.v_cache", + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.29.feed_forward.router", + "language_model.model.layers.29.feed_forward.experts", + "language_model.model.layers.29.feed_forward.experts.moe_op", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.29.feed_forward.shared_expert.down_proj", + "language_model.model.layers.30.self_attn.qkv_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.30.self_attn.attn.impl.softmax", + "language_model.model.layers.30.self_attn.attn.impl.matmul_av", + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.30.self_attn.attn.impl.k_cache", + "language_model.model.layers.30.self_attn.attn.impl.v_cache", + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.30.feed_forward.gate_up_proj", + "language_model.model.layers.30.feed_forward.down_proj", + "language_model.model.layers.31.self_attn.qkv_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.31.self_attn.attn.impl.softmax", + "language_model.model.layers.31.self_attn.attn.impl.matmul_av", + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.31.self_attn.attn.impl.k_cache", + "language_model.model.layers.31.self_attn.attn.impl.v_cache", + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.31.feed_forward.router", + "language_model.model.layers.31.feed_forward.experts", + "language_model.model.layers.31.feed_forward.experts.moe_op", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.31.feed_forward.shared_expert.down_proj", + "language_model.model.layers.32.self_attn.qkv_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.32.self_attn.attn.impl.softmax", + "language_model.model.layers.32.self_attn.attn.impl.matmul_av", + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.32.self_attn.attn.impl.k_cache", + "language_model.model.layers.32.self_attn.attn.impl.v_cache", + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.32.feed_forward.gate_up_proj", + "language_model.model.layers.32.feed_forward.down_proj", + "language_model.model.layers.33.self_attn.qkv_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.33.self_attn.attn.impl.softmax", + "language_model.model.layers.33.self_attn.attn.impl.matmul_av", + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.33.self_attn.attn.impl.k_cache", + "language_model.model.layers.33.self_attn.attn.impl.v_cache", + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.33.feed_forward.router", + "language_model.model.layers.33.feed_forward.experts", + "language_model.model.layers.33.feed_forward.experts.moe_op", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.33.feed_forward.shared_expert.down_proj", + "language_model.model.layers.34.self_attn.qkv_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.34.self_attn.attn.impl.softmax", + "language_model.model.layers.34.self_attn.attn.impl.matmul_av", + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.34.self_attn.attn.impl.k_cache", + "language_model.model.layers.34.self_attn.attn.impl.v_cache", + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.34.feed_forward.gate_up_proj", + "language_model.model.layers.34.feed_forward.down_proj", + "language_model.model.layers.35.self_attn.qkv_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.35.self_attn.attn.impl.softmax", + "language_model.model.layers.35.self_attn.attn.impl.matmul_av", + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.35.self_attn.attn.impl.k_cache", + "language_model.model.layers.35.self_attn.attn.impl.v_cache", + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.35.feed_forward.router", + "language_model.model.layers.35.feed_forward.experts", + "language_model.model.layers.35.feed_forward.experts.moe_op", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.35.feed_forward.shared_expert.down_proj", + "language_model.model.layers.36.self_attn.qkv_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.36.self_attn.attn.impl.softmax", + "language_model.model.layers.36.self_attn.attn.impl.matmul_av", + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.36.self_attn.attn.impl.k_cache", + "language_model.model.layers.36.self_attn.attn.impl.v_cache", + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.36.feed_forward.gate_up_proj", + "language_model.model.layers.36.feed_forward.down_proj", + "language_model.model.layers.37.self_attn.qkv_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.37.self_attn.attn.impl.softmax", + "language_model.model.layers.37.self_attn.attn.impl.matmul_av", + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.37.self_attn.attn.impl.k_cache", + "language_model.model.layers.37.self_attn.attn.impl.v_cache", + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.37.feed_forward.router", + "language_model.model.layers.37.feed_forward.experts", + "language_model.model.layers.37.feed_forward.experts.moe_op", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.37.feed_forward.shared_expert.down_proj", + "language_model.model.layers.38.self_attn.qkv_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.38.self_attn.attn.impl.softmax", + "language_model.model.layers.38.self_attn.attn.impl.matmul_av", + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.38.self_attn.attn.impl.k_cache", + "language_model.model.layers.38.self_attn.attn.impl.v_cache", + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.38.feed_forward.gate_up_proj", + "language_model.model.layers.38.feed_forward.down_proj", + "language_model.model.layers.39.self_attn.qkv_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.39.self_attn.attn.impl.softmax", + "language_model.model.layers.39.self_attn.attn.impl.matmul_av", + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.39.self_attn.attn.impl.k_cache", + "language_model.model.layers.39.self_attn.attn.impl.v_cache", + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.39.feed_forward.router", + "language_model.model.layers.39.feed_forward.experts", + "language_model.model.layers.39.feed_forward.experts.moe_op", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.39.feed_forward.shared_expert.down_proj", + "language_model.model.layers.40.self_attn.qkv_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.40.self_attn.attn.impl.softmax", + "language_model.model.layers.40.self_attn.attn.impl.matmul_av", + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.40.self_attn.attn.impl.k_cache", + "language_model.model.layers.40.self_attn.attn.impl.v_cache", + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.40.feed_forward.gate_up_proj", + "language_model.model.layers.40.feed_forward.down_proj", + "language_model.model.layers.41.self_attn.qkv_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.41.self_attn.attn.impl.softmax", + "language_model.model.layers.41.self_attn.attn.impl.matmul_av", + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.41.self_attn.attn.impl.k_cache", + "language_model.model.layers.41.self_attn.attn.impl.v_cache", + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.41.feed_forward.router", + "language_model.model.layers.41.feed_forward.experts", + "language_model.model.layers.41.feed_forward.experts.moe_op", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.41.feed_forward.shared_expert.down_proj", + "language_model.model.layers.42.self_attn.qkv_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.42.self_attn.attn.impl.softmax", + "language_model.model.layers.42.self_attn.attn.impl.matmul_av", + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.42.self_attn.attn.impl.k_cache", + "language_model.model.layers.42.self_attn.attn.impl.v_cache", + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.42.feed_forward.gate_up_proj", + "language_model.model.layers.42.feed_forward.down_proj", + "language_model.model.layers.43.self_attn.qkv_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.43.self_attn.attn.impl.softmax", + "language_model.model.layers.43.self_attn.attn.impl.matmul_av", + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.43.self_attn.attn.impl.k_cache", + "language_model.model.layers.43.self_attn.attn.impl.v_cache", + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.43.feed_forward.router", + "language_model.model.layers.43.feed_forward.experts", + "language_model.model.layers.43.feed_forward.experts.moe_op", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.43.feed_forward.shared_expert.down_proj", + "language_model.model.layers.44.self_attn.qkv_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.44.self_attn.attn.impl.softmax", + "language_model.model.layers.44.self_attn.attn.impl.matmul_av", + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.44.self_attn.attn.impl.k_cache", + "language_model.model.layers.44.self_attn.attn.impl.v_cache", + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.44.feed_forward.gate_up_proj", + "language_model.model.layers.44.feed_forward.down_proj", + "language_model.model.layers.45.self_attn.qkv_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.45.self_attn.attn.impl.softmax", + "language_model.model.layers.45.self_attn.attn.impl.matmul_av", + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.45.self_attn.attn.impl.k_cache", + "language_model.model.layers.45.self_attn.attn.impl.v_cache", + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.45.feed_forward.router", + "language_model.model.layers.45.feed_forward.experts", + "language_model.model.layers.45.feed_forward.experts.moe_op", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.45.feed_forward.shared_expert.down_proj", + "language_model.model.layers.46.self_attn.qkv_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.46.self_attn.attn.impl.softmax", + "language_model.model.layers.46.self_attn.attn.impl.matmul_av", + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.46.self_attn.attn.impl.k_cache", + "language_model.model.layers.46.self_attn.attn.impl.v_cache", + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.46.feed_forward.gate_up_proj", + "language_model.model.layers.46.feed_forward.down_proj", + "language_model.model.layers.47.self_attn.qkv_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.47.self_attn.attn.impl.softmax", + "language_model.model.layers.47.self_attn.attn.impl.matmul_av", + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.47.self_attn.attn.impl.k_cache", + "language_model.model.layers.47.self_attn.attn.impl.v_cache", + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.47.feed_forward.router", + "language_model.model.layers.47.feed_forward.experts", + "language_model.model.layers.47.feed_forward.experts.moe_op", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.47.feed_forward.shared_expert.down_proj", + "language_model.lm_head" +] \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_0_8.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_0_8.json new file mode 100644 index 000000000000..7be5f0ec51e1 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_0_8.json @@ -0,0 +1 @@ +{"GlobalRank": null, "LocalRank": 0, "Mode": "DynamicRange", "Nodes": {"language_model.model.layers.0.self_attn.qkv_proj": {"inputs": [[[11.1875]]], "params": {"weight": [[0.51171875]]}}, "language_model.model.layers.0.self_attn.o_proj": {"inputs": [[[0.640625]]], "outputs": [[[0.80078125]], [[3.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.0.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.55859375]], [[5.59375]]]}, "language_model.model.layers.0.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.640625]]]}, "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.55859375]]]}, "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.640625]]]}, "language_model.model.layers.0.self_attn.attn.impl.k_cache": {"inputs": [[[5.59375]]]}, "language_model.model.layers.0.self_attn.attn.impl.v_cache": {"inputs": [[[0.640625]]]}, "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[7.125]], [[5.59375]], [[0.640625]]], "outputs": [[[0.640625]], [[1.0]]]}, "language_model.model.layers.0.feed_forward.gate_up_proj": {"inputs": [[[2.546875]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.0.feed_forward.down_proj": {"inputs": [[[37.25]]], "outputs": [[[26.75]], [[65.0]]], "params": {"weight": [[0.734375]]}}, "language_model.model.layers.1.self_attn.qkv_proj": {"inputs": [[[12.3125]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.1.self_attn.o_proj": {"inputs": [[[0.92578125]]], "outputs": [[[0.52734375]], [[1.0078125]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.1.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.69140625]], [[9.5625]]]}, "language_model.model.layers.1.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.0546875]]]}, "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.69140625]]]}, "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.8671875]]]}, "language_model.model.layers.1.self_attn.attn.impl.k_cache": {"inputs": [[[9.5625]]]}, "language_model.model.layers.1.self_attn.attn.impl.v_cache": {"inputs": [[[1.0546875]]]}, "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.125]], [[9.5]], [[1.0546875]]], "outputs": [[[0.92578125]], [[1.0]]]}, "language_model.model.layers.1.feed_forward.router": {"inputs": [[[1.8828125]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op": {"inputs": [[[1.8828125]]], "outputs": [[[1.046875]], [[12.125]], [[0.0]], [[0.66015625]], [[12.25]], [[12.125]], [[15.5]], [[12.125]], [[8.3125]], [[13.5]], [[7.625]], [[13.3125]], [[12.375]], [[2.84375]], [[8.4375]], [[14.25]], [[17.75]], [[14.0]], [[6.78125]], [[12.4375]], [[14.875]], [[11.25]], [[10.9375]], [[0.0]], [[16.375]], [[5.4375]], [[0.58984375]], [[10.3125]], [[10.3125]], [[8.0]], [[13.1875]], [[15.1875]], [[17.875]], [[0.216796875]], [[5.34375]], [[0.0]], [[17.25]], [[9.0]], [[12.3125]], [[3.984375]], [[13.4375]], [[10.375]], [[12.625]], [[12.75]], [[11.25]], [[4.6875]], [[11.8125]], [[12.9375]], [[5.78125]], [[2.09375]], [[0.0]], [[8.75]], [[10.9375]], [[8.125]], [[9.1875]], [[7.65625]], [[15.9375]], [[10.1875]], [[0.52734375]], [[9.8125]], [[12.3125]], [[12.25]], [[12.5625]], [[14.0625]], [[0.0]], [[12.75]], [[5.28125]], [[2.125]], [[11.8125]], [[3.40625]], [[5.96875]], [[13.0]], [[6.9375]], [[15.1875]], [[18.125]], [[9.5]], [[10.125]], [[14.75]], [[12.0]], [[3.203125]], [[13.5]], [[0.0]], [[1.1328125]], [[13.375]], [[2.453125]], [[13.5625]], [[8.5625]], [[13.375]], [[13.8125]], [[6.59375]], [[9.75]], [[12.8125]], [[9.1875]], [[18.375]], [[0.0]], [[12.3125]], [[10.25]], [[0.07666015625]], [[0.0]], [[13.5]], [[13.0]], [[11.25]], [[2.3125]], [[8.9375]], [[8.9375]], [[2.546875]], [[5.5625]], [[0.08056640625]], [[0.625]], [[7.8125]], [[0.1826171875]], [[8.125]], [[0.86328125]], [[9.4375]], [[0.0]], [[14.25]], [[11.0]], [[11.5]], [[3.109375]], [[15.9375]], [[6.25]], [[10.25]], [[12.5]], [[3.53125]], [[4.46875]], [[8.75]], [[14.8125]], [[10.9375]], [[12.5625]]]}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.45703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.07080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0751953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[1.8828125]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.1.feed_forward.shared_expert.down_proj": {"inputs": [[[5.8125]]], "outputs": [[[0.37890625]], [[2.8744555408848806e+37]]], "params": {"weight": [[0.365234375]]}}, "language_model.model.layers.2.self_attn.qkv_proj": {"inputs": [[[9.875]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.2.self_attn.o_proj": {"inputs": [[[0.455078125]]], "outputs": [[[0.6875]], [[1.328125]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.2.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.7890625]], [[12.6875]]]}, "language_model.model.layers.2.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.984375]]]}, "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.7890625]]]}, "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.68359375]]]}, "language_model.model.layers.2.self_attn.attn.impl.k_cache": {"inputs": [[[12.6875]]]}, "language_model.model.layers.2.self_attn.attn.impl.v_cache": {"inputs": [[[0.984375]]]}, "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.75]], [[12.6875]], [[0.94921875]]], "outputs": [[[0.455078125]], [[1.0]]]}, "language_model.model.layers.2.feed_forward.gate_up_proj": {"inputs": [[[6.125]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.2.feed_forward.down_proj": {"inputs": [[[5.3125]]], "outputs": [[[4.75]], [[25.25]]], "params": {"weight": [[0.498046875]]}}, "language_model.model.layers.3.self_attn.qkv_proj": {"inputs": [[[7.21875]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.3.self_attn.o_proj": {"inputs": [[[0.294921875]]], "outputs": [[[0.59765625]], [[2.40625]]], "params": {"weight": [[0.419921875]]}}, "language_model.model.layers.3.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.0615234375]], [[3.4375]]]}, "language_model.model.layers.3.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.40234375]]]}, "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.0615234375]]]}, "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.34765625]]]}, "language_model.model.layers.3.self_attn.attn.impl.k_cache": {"inputs": [[[3.4375]]]}, "language_model.model.layers.3.self_attn.attn.impl.v_cache": {"inputs": [[[0.40234375]]]}, "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[1.1484375]], [[3.4375]], [[0.40234375]]], "outputs": [[[0.294921875]], [[1.0]]]}, "language_model.model.layers.3.feed_forward.router": {"inputs": [[[8.75]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op": {"inputs": [[[8.75]]], "outputs": [[[336.0]], [[0.310546875]], [[8.75]], [[18.75]], [[3.84375]], [[0.0]], [[5.0625]], [[13.625]], [[13.875]], [[12.625]], [[14.3125]], [[4.28125]], [[8.375]], [[12.5625]], [[8.4375]], [[9.625]], [[16.875]], [[9.5625]], [[1.578125]], [[8.875]], [[13.375]], [[4.1875]], [[12.625]], [[16.0]], [[19.5]], [[9.6875]], [[1.8671875]], [[13.75]], [[6.25]], [[10.25]], [[12.0]], [[14.25]], [[10.125]], [[7.78125]], [[9.6875]], [[12.875]], [[11.3125]], [[3.75]], [[3.34375]], [[14.4375]], [[6.71875]], [[1.2265625]], [[13.6875]], [[0.76171875]], [[2.28125]], [[20.375]], [[6.78125]], [[14.875]], [[12.1875]], [[5.46875]], [[12.0]], [[15.0625]], [[12.625]], [[13.25]], [[2.421875]], [[6.75]], [[17.5]], [[2.640625]], [[12.0]], [[20.25]], [[12.6875]], [[5.28125]], [[16.125]], [[14.375]], [[0.248046875]], [[9.0625]], [[9.875]], [[18.5]], [[22.75]], [[1.8046875]], [[0.0]], [[14.25]], [[11.4375]], [[9.6875]], [[4.09375]], [[2.28125]], [[6.28125]], [[12.6875]], [[0.1318359375]], [[17.5]], [[10.5]], [[10.1875]], [[0.0]], [[6.0]], [[173.0]], [[10.0]], [[17.875]], [[5.46875]], [[1.1796875]], [[14.3125]], [[18.75]], [[16.625]], [[5.53125]], [[3.953125]], [[15.375]], [[0.0]], [[11.6875]], [[13.1875]], [[11.4375]], [[7.875]], [[1.6875]], [[15.5]], [[5.40625]], [[4.0]], [[15.1875]], [[0.0]], [[6.21875]], [[16.25]], [[16.125]], [[11.0625]], [[4.25]], [[14.5]], [[2.75]], [[10.6875]], [[4.21875]], [[2.75]], [[5.75]], [[8.1875]], [[0.62109375]], [[3.703125]], [[2.859375]], [[18.75]], [[14.0]], [[7.8125]], [[12.3125]], [[4.25]], [[18.125]], [[11.25]], [[10.9375]]]}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.75]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[8.75]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.3.feed_forward.shared_expert.down_proj": {"inputs": [[[6.65625]]], "outputs": [[[0.76171875]], [[2.924301590726815e+38]]], "params": {"weight": [[0.5234375]]}}, "language_model.model.layers.4.self_attn.qkv_proj": {"inputs": [[[13.75]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.4.self_attn.o_proj": {"inputs": [[[1.0]]], "outputs": [[[1.015625]], [[2.046875]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.4.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.87890625]], [[14.6875]]]}, "language_model.model.layers.4.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.796875]]]}, "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.87890625]]]}, "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.7890625]]]}, "language_model.model.layers.4.self_attn.attn.impl.k_cache": {"inputs": [[[14.6875]]]}, "language_model.model.layers.4.self_attn.attn.impl.v_cache": {"inputs": [[[2.796875]]]}, "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.5]], [[14.6875]], [[2.796875]]], "outputs": [[[1.0]], [[1.0]]]}, "language_model.model.layers.4.feed_forward.gate_up_proj": {"inputs": [[[7.09375]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.4.feed_forward.down_proj": {"inputs": [[[7.15625]]], "outputs": [[[2.796875]], [[4.96875]]], "params": {"weight": [[0.58984375]]}}, "language_model.model.layers.5.self_attn.qkv_proj": {"inputs": [[[17.375]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.5.self_attn.o_proj": {"inputs": [[[1.0703125]]], "outputs": [[[0.97265625]], [[2.21875]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.5.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.86328125]], [[17.25]]]}, "language_model.model.layers.5.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.328125]]]}, "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.86328125]]]}, "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.6484375]]]}, "language_model.model.layers.5.self_attn.attn.impl.k_cache": {"inputs": [[[17.25]]]}, "language_model.model.layers.5.self_attn.attn.impl.v_cache": {"inputs": [[[2.328125]]]}, "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.875]], [[17.25]], [[2.328125]]], "outputs": [[[1.0703125]], [[1.0]]]}, "language_model.model.layers.5.feed_forward.router": {"inputs": [[[19.25]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op": {"inputs": [[[19.25]]], "outputs": [[[350.0]], [[8.0625]], [[3.40625]], [[10.875]], [[15.875]], [[12.1875]], [[13.6875]], [[18.375]], [[21.5]], [[10.0]], [[17.125]], [[11.0]], [[22.75]], [[12.75]], [[5.40625]], [[13.125]], [[23.875]], [[9.5625]], [[6.9375]], [[23.75]], [[4.0625]], [[24.375]], [[8.25]], [[9.4375]], [[17.375]], [[2.46875]], [[2.09375]], [[15.25]], [[226.0]], [[9.8125]], [[25.125]], [[22.25]], [[5.71875]], [[8.75]], [[30.25]], [[8.375]], [[19.125]], [[21.25]], [[5.4375]], [[10.5]], [[17.75]], [[13.375]], [[3.671875]], [[3.53125]], [[14.0]], [[12.0625]], [[16.875]], [[7.84375]], [[18.0]], [[4.28125]], [[12.0]], [[5.59375]], [[20.25]], [[1.8984375]], [[9.9375]], [[10.3125]], [[3.90625]], [[12.1875]], [[19.75]], [[15.375]], [[0.8828125]], [[10.0]], [[17.875]], [[3.484375]], [[15.1875]], [[4.75]], [[11.4375]], [[6.65625]], [[7.03125]], [[18.625]], [[10.3125]], [[5.125]], [[13.75]], [[16.5]], [[12.1875]], [[4.53125]], [[21.125]], [[15.5625]], [[7.96875]], [[24.125]], [[20.625]], [[1.9296875]], [[12.1875]], [[6.375]], [[19.5]], [[10.625]], [[12.4375]], [[0.1689453125]], [[14.875]], [[14.125]], [[17.125]], [[20.0]], [[15.0]], [[14.0625]], [[24.625]], [[18.375]], [[2.515625]], [[15.75]], [[5.0625]], [[3.4375]], [[13.625]], [[5.96875]], [[2.640625]], [[3.359375]], [[19.625]], [[20.125]], [[6.8125]], [[7.15625]], [[16.875]], [[13.125]], [[10.5625]], [[18.875]], [[11.875]], [[4.5]], [[12.0]], [[15.5]], [[8.5]], [[2.078125]], [[3.703125]], [[3.796875]], [[14.6875]], [[15.5625]], [[11.1875]], [[9.375]], [[27.5]], [[17.25]], [[1.390625]], [[21.125]], [[11.0625]]]}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.431640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.47265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.77734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.69921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[19.25]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.5.feed_forward.shared_expert.down_proj": {"inputs": [[[5.96875]]], "outputs": [[[1.1015625]], [[3.017347550431759e+38]]], "params": {"weight": [[0.35546875]]}}, "language_model.model.layers.6.self_attn.qkv_proj": {"inputs": [[[12.9375]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.6.self_attn.o_proj": {"inputs": [[[1.6484375]]], "outputs": [[[0.66796875]], [[1.5234375]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.6.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.890625]], [[15.25]]]}, "language_model.model.layers.6.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.546875]]]}, "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.890625]]]}, "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.8046875]]]}, "language_model.model.layers.6.self_attn.attn.impl.k_cache": {"inputs": [[[15.25]]]}, "language_model.model.layers.6.self_attn.attn.impl.v_cache": {"inputs": [[[2.546875]]]}, "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.5625]], [[15.25]], [[2.546875]]], "outputs": [[[1.6484375]], [[1.0]]]}, "language_model.model.layers.6.feed_forward.gate_up_proj": {"inputs": [[[7.71875]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.6.feed_forward.down_proj": {"inputs": [[[8.5]]], "outputs": [[[2.328125]], [[11.1875]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.7.self_attn.qkv_proj": {"inputs": [[[7.09375]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.7.self_attn.o_proj": {"inputs": [[[0.6015625]]], "outputs": [[[1.84375]], [[3.84375]]], "params": {"weight": [[0.37109375]]}}, "language_model.model.layers.7.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.2099609375]], [[3.609375]]]}, "language_model.model.layers.7.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.296875]]]}, "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.2099609375]]]}, "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.97265625]]]}, "language_model.model.layers.7.self_attn.attn.impl.k_cache": {"inputs": [[[3.609375]]]}, "language_model.model.layers.7.self_attn.attn.impl.v_cache": {"inputs": [[[1.296875]]]}, "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[2.65625]], [[3.484375]], [[1.296875]]], "outputs": [[[0.6015625]], [[1.0]]]}, "language_model.model.layers.7.feed_forward.router": {"inputs": [[[11.6875]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op": {"inputs": [[[11.6875]]], "outputs": [[[2.78125]], [[11.6875]], [[10.5625]], [[17.375]], [[15.5625]], [[9.6875]], [[12.1875]], [[10.75]], [[13.75]], [[11.375]], [[18.125]], [[35.25]], [[12.75]], [[7.71875]], [[22.75]], [[6.625]], [[4.0]], [[17.75]], [[6.71875]], [[20.5]], [[14.0625]], [[22.25]], [[22.625]], [[34.0]], [[24.125]], [[19.5]], [[10.4375]], [[8.0625]], [[6.65625]], [[14.625]], [[12.0625]], [[9.625]], [[9.1875]], [[6.21875]], [[6.75]], [[6.09375]], [[15.5]], [[13.4375]], [[20.5]], [[36.75]], [[1.0390625]], [[1.15625]], [[5.875]], [[1.1796875]], [[17.75]], [[10.8125]], [[26.75]], [[0.734375]], [[24.375]], [[25.25]], [[21.125]], [[16.75]], [[17.0]], [[21.375]], [[16.375]], [[25.5]], [[21.25]], [[5.25]], [[2.53125]], [[3.578125]], [[5.4375]], [[10.875]], [[20.125]], [[9.4375]], [[8.8125]], [[27.875]], [[12.0625]], [[11.6875]], [[5.53125]], [[6.8125]], [[7.28125]], [[30.25]], [[2.984375]], [[10.5625]], [[18.25]], [[24.625]], [[4.3125]], [[4.03125]], [[6.96875]], [[4.84375]], [[7.09375]], [[21.75]], [[12.5]], [[2.828125]], [[8.25]], [[3.65625]], [[16.875]], [[2.65625]], [[30.875]], [[3.125]], [[34.75]], [[26.625]], [[11.1875]], [[6.875]], [[3.859375]], [[24.25]], [[8.375]], [[19.375]], [[11.1875]], [[47.75]], [[6.71875]], [[13.0625]], [[14.9375]], [[4.125]], [[0.58984375]], [[21.5]], [[28.5]], [[16.375]], [[6.9375]], [[1.6875]], [[14.4375]], [[18.875]], [[21.75]], [[20.25]], [[27.0]], [[22.25]], [[2.53125]], [[7.40625]], [[13.3125]], [[2.15625]], [[37.5]], [[20.625]], [[25.875]], [[19.375]], [[7.6875]], [[7.78125]], [[2.078125]], [[24.875]], [[12.625]]]}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.5859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.34765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.55859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[11.6875]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.7.feed_forward.shared_expert.down_proj": {"inputs": [[[10.625]]], "outputs": [[[1.0390625]], [[3.349654549377988e+38]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.8.self_attn.qkv_proj": {"inputs": [[[15.125]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.8.self_attn.o_proj": {"inputs": [[[1.9765625]]], "outputs": [[[1.9375]], [[2.109375]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.8.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.890625]], [[16.375]]]}, "language_model.model.layers.8.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.328125]]]}, "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.890625]]]}, "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.0625]]]}, "language_model.model.layers.8.self_attn.attn.impl.k_cache": {"inputs": [[[16.375]]]}, "language_model.model.layers.8.self_attn.attn.impl.v_cache": {"inputs": [[[3.328125]]]}, "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.5625]], [[16.375]], [[3.328125]]], "outputs": [[[1.9765625]], [[1.0]]]}, "language_model.model.layers.8.feed_forward.gate_up_proj": {"inputs": [[[8.25]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.8.feed_forward.down_proj": {"inputs": [[[10.0]]], "outputs": [[[2.5625]], [[9.25]]], "params": {"weight": [[0.48828125]]}}, "language_model.model.layers.9.self_attn.qkv_proj": {"inputs": [[[15.75]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.9.self_attn.o_proj": {"inputs": [[[2.421875]]], "outputs": [[[1.3125]], [[2.875]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.9.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.87890625]], [[13.375]]]}, "language_model.model.layers.9.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.0625]]]}, "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.87890625]]]}, "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.3125]]]}, "language_model.model.layers.9.self_attn.attn.impl.k_cache": {"inputs": [[[13.375]]]}, "language_model.model.layers.9.self_attn.attn.impl.v_cache": {"inputs": [[[3.0625]]]}, "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.875]], [[13.375]], [[3.0625]]], "outputs": [[[2.421875]], [[1.0]]]}, "language_model.model.layers.9.feed_forward.router": {"inputs": [[[11.75]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op": {"inputs": [[[11.75]]], "outputs": [[[3.390625]], [[22.25]], [[13.0]], [[4.40625]], [[36.5]], [[28.75]], [[15.6875]], [[42.0]], [[24.5]], [[22.5]], [[30.375]], [[18.625]], [[12.5625]], [[52.25]], [[21.375]], [[6.46875]], [[5.46875]], [[35.5]], [[41.25]], [[27.75]], [[15.125]], [[22.25]], [[18.875]], [[6.4375]], [[18.75]], [[13.75]], [[15.0]], [[29.875]], [[41.75]], [[14.3125]], [[38.5]], [[14.6875]], [[14.125]], [[36.25]], [[31.375]], [[30.25]], [[7.125]], [[28.625]], [[25.625]], [[8.5625]], [[17.875]], [[10.4375]], [[8.875]], [[25.125]], [[27.5]], [[9.125]], [[12.0625]], [[20.0]], [[29.5]], [[25.875]], [[37.5]], [[8.875]], [[26.5]], [[17.875]], [[12.8125]], [[28.0]], [[11.625]], [[39.75]], [[30.0]], [[35.0]], [[27.5]], [[20.625]], [[16.875]], [[18.5]], [[32.0]], [[19.375]], [[21.375]], [[38.75]], [[12.125]], [[11.8125]], [[17.0]], [[16.875]], [[33.75]], [[13.9375]], [[12.8125]], [[9.1875]], [[18.375]], [[26.875]], [[19.375]], [[5.0]], [[10.1875]], [[12.5]], [[25.5]], [[29.0]], [[32.5]], [[32.5]], [[31.875]], [[10.9375]], [[21.625]], [[27.75]], [[35.0]], [[13.1875]], [[7.4375]], [[18.25]], [[20.625]], [[26.375]], [[29.375]], [[11.5]], [[12.0]], [[9.5625]], [[14.6875]], [[15.5]], [[15.25]], [[41.75]], [[14.9375]], [[25.375]], [[11.75]], [[26.625]], [[15.75]], [[23.625]], [[37.0]], [[23.625]], [[3.0]], [[23.25]], [[9.6875]], [[22.25]], [[28.375]], [[16.5]], [[6.71875]], [[26.25]], [[25.375]], [[57.0]], [[27.125]], [[33.25]], [[30.625]], [[38.75]], [[23.125]], [[11.0625]], [[23.0]]]}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.39453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.63671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[11.75]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.9.feed_forward.shared_expert.down_proj": {"inputs": [[[11.4375]]], "outputs": [[[0.6953125]], [[3.349654549377988e+38]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.10.self_attn.qkv_proj": {"inputs": [[[16.125]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.10.self_attn.o_proj": {"inputs": [[[3.0]]], "outputs": [[[1.6015625]], [[3.09375]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.10.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.71875]], [[14.5]]]}, "language_model.model.layers.10.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.90625]]]}, "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.71875]]]}, "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.703125]]]}, "language_model.model.layers.10.self_attn.attn.impl.k_cache": {"inputs": [[[14.5625]]]}, "language_model.model.layers.10.self_attn.attn.impl.v_cache": {"inputs": [[[3.90625]]]}, "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.875]], [[14.5625]], [[3.90625]]], "outputs": [[[3.0]], [[1.0]]]}, "language_model.model.layers.10.feed_forward.gate_up_proj": {"inputs": [[[9.875]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.10.feed_forward.down_proj": {"inputs": [[[11.625]]], "outputs": [[[6.90625]], [[11.3125]]], "params": {"weight": [[0.408203125]]}}, "language_model.model.layers.11.self_attn.qkv_proj": {"inputs": [[[13.5625]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.11.self_attn.o_proj": {"inputs": [[[3.65625]]], "outputs": [[[8.0625]], [[7.875]]], "params": {"weight": [[0.375]]}}, "language_model.model.layers.11.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.6171875]], [[8.25]]]}, "language_model.model.layers.11.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.75]]]}, "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.6171875]]]}, "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.15625]]]}, "language_model.model.layers.11.self_attn.attn.impl.k_cache": {"inputs": [[[8.25]]]}, "language_model.model.layers.11.self_attn.attn.impl.v_cache": {"inputs": [[[4.75]]]}, "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.3125]], [[8.25]], [[4.75]]], "outputs": [[[3.65625]], [[1.0]]]}, "language_model.model.layers.11.feed_forward.router": {"inputs": [[[13.375]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op": {"inputs": [[[13.375]]], "outputs": [[[4.59375]], [[23.5]], [[19.875]], [[19.625]], [[13.0]], [[17.5]], [[22.875]], [[1.5546875]], [[17.625]], [[23.125]], [[10.3125]], [[28.875]], [[16.375]], [[37.5]], [[40.0]], [[18.375]], [[54.0]], [[52.25]], [[43.75]], [[53.5]], [[13.0]], [[23.0]], [[34.5]], [[28.875]], [[35.25]], [[20.5]], [[59.75]], [[26.375]], [[13.75]], [[10.25]], [[68.5]], [[18.5]], [[5.59375]], [[12.625]], [[13.5625]], [[26.125]], [[17.375]], [[16.375]], [[20.25]], [[15.75]], [[17.25]], [[3.828125]], [[9.125]], [[19.375]], [[33.0]], [[25.75]], [[1.96875]], [[27.25]], [[30.375]], [[23.625]], [[14.125]], [[38.25]], [[53.0]], [[4.46875]], [[52.5]], [[56.5]], [[45.5]], [[17.375]], [[19.125]], [[29.75]], [[20.625]], [[43.25]], [[13.6875]], [[41.0]], [[23.125]], [[16.125]], [[22.5]], [[32.75]], [[51.0]], [[12.375]], [[42.25]], [[5.25]], [[13.1875]], [[6.1875]], [[30.375]], [[16.75]], [[17.625]], [[17.75]], [[3.40625]], [[54.0]], [[30.5]], [[22.5]], [[23.75]], [[26.5]], [[11.8125]], [[1.6328125]], [[2.734375]], [[30.75]], [[15.5]], [[9.1875]], [[24.375]], [[43.5]], [[10.6875]], [[22.5]], [[45.25]], [[10.5625]], [[18.0]], [[20.625]], [[34.0]], [[13.375]], [[28.75]], [[9.0625]], [[54.5]], [[31.375]], [[18.875]], [[47.5]], [[14.75]], [[10.0625]], [[11.125]], [[16.875]], [[14.1875]], [[31.75]], [[6.03125]], [[4.9375]], [[10.5]], [[34.25]], [[34.0]], [[34.25]], [[33.25]], [[22.125]], [[28.75]], [[27.625]], [[28.375]], [[19.125]], [[13.8125]], [[9.3125]], [[8.3125]], [[44.0]], [[21.75]]]}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[13.375]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.11.feed_forward.shared_expert.down_proj": {"inputs": [[[11.8125]]], "outputs": [[[2.890625]], [[3.017347550431759e+38]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.12.self_attn.qkv_proj": {"inputs": [[[18.625]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.12.self_attn.o_proj": {"inputs": [[[1.7734375]]], "outputs": [[[1.3359375]], [[3.640625]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.12.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.95703125]], [[18.0]]]}, "language_model.model.layers.12.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.75]]]}, "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.95703125]]]}, "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.8125]]]}, "language_model.model.layers.12.self_attn.attn.impl.k_cache": {"inputs": [[[18.0]]]}, "language_model.model.layers.12.self_attn.attn.impl.v_cache": {"inputs": [[[2.75]]]}, "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.0625]], [[18.0]], [[2.75]]], "outputs": [[[1.7734375]], [[1.0]]]}, "language_model.model.layers.12.feed_forward.gate_up_proj": {"inputs": [[[10.75]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.12.feed_forward.down_proj": {"inputs": [[[20.75]]], "outputs": [[[9.0]], [[17.375]]], "params": {"weight": [[0.68359375]]}}, "language_model.model.layers.13.self_attn.qkv_proj": {"inputs": [[[20.875]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.13.self_attn.o_proj": {"inputs": [[[3.203125]]], "outputs": [[[1.6875]], [[6.9375]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.13.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.72265625]], [[12.25]]]}, "language_model.model.layers.13.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.75]]]}, "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.72265625]]]}, "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.96875]]]}, "language_model.model.layers.13.self_attn.attn.impl.k_cache": {"inputs": [[[12.25]]]}, "language_model.model.layers.13.self_attn.attn.impl.v_cache": {"inputs": [[[3.75]]]}, "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.1875]], [[12.25]], [[3.75]]], "outputs": [[[3.203125]], [[1.0]]]}, "language_model.model.layers.13.feed_forward.router": {"inputs": [[[17.625]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op": {"inputs": [[[17.625]]], "outputs": [[[4.0625]], [[31.0]], [[50.5]], [[31.5]], [[55.0]], [[34.5]], [[8.125]], [[10.5]], [[27.875]], [[12.1875]], [[2.46875]], [[28.25]], [[37.0]], [[8.875]], [[25.0]], [[38.0]], [[28.5]], [[22.125]], [[41.0]], [[12.375]], [[23.0]], [[20.625]], [[44.75]], [[7.5625]], [[2.34375]], [[15.6875]], [[62.75]], [[27.75]], [[60.5]], [[63.75]], [[8.4375]], [[11.1875]], [[16.5]], [[31.25]], [[10.625]], [[25.25]], [[17.625]], [[35.5]], [[57.25]], [[22.375]], [[5.625]], [[31.75]], [[27.25]], [[20.75]], [[22.25]], [[42.0]], [[49.75]], [[22.375]], [[11.875]], [[33.5]], [[61.0]], [[31.0]], [[51.5]], [[8.5625]], [[63.5]], [[3.5]], [[34.25]], [[35.75]], [[20.625]], [[45.0]], [[27.375]], [[20.375]], [[52.0]], [[9.75]], [[48.0]], [[20.875]], [[39.5]], [[5.28125]], [[28.5]], [[30.375]], [[11.6875]], [[46.75]], [[51.5]], [[8.125]], [[16.75]], [[30.875]], [[29.125]], [[37.25]], [[26.875]], [[18.875]], [[20.875]], [[10.375]], [[36.25]], [[3.046875]], [[44.25]], [[38.5]], [[52.5]], [[23.75]], [[10.75]], [[3.65625]], [[39.75]], [[52.5]], [[54.0]], [[3.078125]], [[42.75]], [[51.0]], [[8.0]], [[15.9375]], [[12.8125]], [[17.875]], [[33.25]], [[5.59375]], [[2.015625]], [[31.625]], [[24.75]], [[35.75]], [[24.25]], [[33.75]], [[15.0625]], [[15.375]], [[44.0]], [[27.75]], [[29.875]], [[12.875]], [[17.125]], [[23.625]], [[43.25]], [[18.0]], [[41.0]], [[49.25]], [[29.75]], [[37.75]], [[41.5]], [[23.375]], [[21.25]], [[26.625]], [[13.3125]], [[10.875]], [[46.75]]]}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.5546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[17.625]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.13.feed_forward.shared_expert.down_proj": {"inputs": [[[29.625]]], "outputs": [[[2.40625]], [[3.349654549377988e+38]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.14.self_attn.qkv_proj": {"inputs": [[[18.75]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.14.self_attn.o_proj": {"inputs": [[[2.40625]]], "outputs": [[[2.140625]], [[4.875]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.14.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8046875]], [[14.625]]]}, "language_model.model.layers.14.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.46875]]]}, "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8046875]]]}, "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.40625]]]}, "language_model.model.layers.14.self_attn.attn.impl.k_cache": {"inputs": [[[14.625]]]}, "language_model.model.layers.14.self_attn.attn.impl.v_cache": {"inputs": [[[3.46875]]]}, "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.5]], [[14.625]], [[3.46875]]], "outputs": [[[2.3125]], [[1.0]]]}, "language_model.model.layers.14.feed_forward.gate_up_proj": {"inputs": [[[12.0625]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.14.feed_forward.down_proj": {"inputs": [[[12.5625]]], "outputs": [[[4.65625]], [[28.25]]], "params": {"weight": [[0.470703125]]}}, "language_model.model.layers.15.self_attn.qkv_proj": {"inputs": [[[20.875]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.15.self_attn.o_proj": {"inputs": [[[3.734375]]], "outputs": [[[21.875]], [[23.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.15.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.85546875]], [[13.8125]]]}, "language_model.model.layers.15.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.5625]]]}, "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.85546875]]]}, "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.53125]]]}, "language_model.model.layers.15.self_attn.attn.impl.k_cache": {"inputs": [[[13.8125]]]}, "language_model.model.layers.15.self_attn.attn.impl.v_cache": {"inputs": [[[4.5625]]]}, "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.8125]], [[13.8125]], [[4.5625]]], "outputs": [[[3.734375]], [[1.0]]]}, "language_model.model.layers.15.feed_forward.router": {"inputs": [[[19.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op": {"inputs": [[[19.0]]], "outputs": [[[5.5]], [[5.4375]], [[14.25]], [[83.5]], [[15.0625]], [[29.875]], [[57.5]], [[31.875]], [[20.0]], [[43.0]], [[58.75]], [[20.125]], [[16.375]], [[15.75]], [[21.875]], [[4.375]], [[19.625]], [[24.625]], [[24.0]], [[16.625]], [[10.1875]], [[42.75]], [[25.125]], [[42.25]], [[51.25]], [[9.5625]], [[16.75]], [[23.75]], [[26.75]], [[40.0]], [[19.375]], [[38.0]], [[32.75]], [[20.75]], [[20.75]], [[41.75]], [[20.375]], [[11.25]], [[31.125]], [[58.25]], [[30.625]], [[43.0]], [[60.5]], [[69.5]], [[48.5]], [[37.0]], [[23.375]], [[7.5]], [[18.875]], [[11.5625]], [[19.75]], [[21.625]], [[46.25]], [[33.75]], [[20.5]], [[33.0]], [[29.25]], [[43.75]], [[23.625]], [[24.0]], [[30.5]], [[16.375]], [[51.25]], [[35.25]], [[56.0]], [[14.5]], [[49.75]], [[40.25]], [[53.25]], [[14.5625]], [[68.5]], [[34.5]], [[11.0]], [[31.5]], [[75.5]], [[50.5]], [[36.5]], [[55.5]], [[36.0]], [[42.0]], [[6.09375]], [[31.625]], [[28.375]], [[28.375]], [[7.59375]], [[8.75]], [[31.375]], [[34.25]], [[14.125]], [[36.0]], [[29.75]], [[6.53125]], [[25.0]], [[22.75]], [[42.0]], [[46.5]], [[40.5]], [[48.0]], [[24.875]], [[14.3125]], [[35.25]], [[59.0]], [[35.75]], [[45.25]], [[20.125]], [[27.625]], [[34.5]], [[15.3125]], [[42.5]], [[39.0]], [[18.125]], [[53.25]], [[48.75]], [[52.75]], [[8.375]], [[13.25]], [[20.25]], [[61.5]], [[18.75]], [[22.5]], [[37.25]], [[54.75]], [[44.25]], [[27.125]], [[56.0]], [[38.25]], [[11.8125]], [[14.5625]], [[43.75]]]}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[19.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.15.feed_forward.shared_expert.down_proj": {"inputs": [[[26.25]]], "outputs": [[[3.078125]], [[3.349654549377988e+38]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.16.self_attn.qkv_proj": {"inputs": [[[20.375]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.16.self_attn.o_proj": {"inputs": [[[5.09375]]], "outputs": [[[2.75]], [[6.28125]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.16.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.15625]], [[17.375]]]}, "language_model.model.layers.16.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.34375]]]}, "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.15625]]]}, "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.53125]]]}, "language_model.model.layers.16.self_attn.attn.impl.k_cache": {"inputs": [[[17.375]]]}, "language_model.model.layers.16.self_attn.attn.impl.v_cache": {"inputs": [[[5.34375]]]}, "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.125]], [[17.375]], [[5.34375]]], "outputs": [[[5.09375]], [[1.0]]]}, "language_model.model.layers.16.feed_forward.gate_up_proj": {"inputs": [[[14.8125]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.16.feed_forward.down_proj": {"inputs": [[[16.625]]], "outputs": [[[7.03125]], [[22.625]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.17.self_attn.qkv_proj": {"inputs": [[[19.875]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.17.self_attn.o_proj": {"inputs": [[[4.1875]]], "outputs": [[[17.5]], [[20.75]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.17.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.015625]], [[15.5]]]}, "language_model.model.layers.17.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.6875]]]}, "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.015625]]]}, "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.546875]]]}, "language_model.model.layers.17.self_attn.attn.impl.k_cache": {"inputs": [[[15.5]]]}, "language_model.model.layers.17.self_attn.attn.impl.v_cache": {"inputs": [[[4.6875]]]}, "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.0625]], [[15.5]], [[4.6875]]], "outputs": [[[4.1875]], [[1.0]]]}, "language_model.model.layers.17.feed_forward.router": {"inputs": [[[23.875]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op": {"inputs": [[[23.875]]], "outputs": [[[24.625]], [[26.75]], [[16.5]], [[27.875]], [[7.125]], [[45.0]], [[41.5]], [[92.0]], [[25.25]], [[47.0]], [[5.90625]], [[12.4375]], [[27.625]], [[18.625]], [[29.0]], [[29.875]], [[45.25]], [[33.5]], [[32.0]], [[29.875]], [[31.5]], [[44.75]], [[40.75]], [[78.5]], [[55.25]], [[9.8125]], [[26.75]], [[17.625]], [[62.0]], [[43.75]], [[42.5]], [[49.5]], [[25.875]], [[9.4375]], [[14.0]], [[29.875]], [[64.0]], [[27.375]], [[29.625]], [[15.375]], [[40.0]], [[22.375]], [[67.0]], [[7.46875]], [[30.25]], [[29.5]], [[42.75]], [[61.75]], [[48.5]], [[46.75]], [[24.625]], [[63.75]], [[37.75]], [[26.75]], [[12.6875]], [[17.0]], [[4.09375]], [[20.875]], [[43.25]], [[27.5]], [[23.625]], [[56.75]], [[46.0]], [[10.5]], [[9.5625]], [[38.5]], [[75.5]], [[13.375]], [[67.5]], [[34.5]], [[19.75]], [[27.75]], [[20.375]], [[53.75]], [[47.75]], [[15.0625]], [[19.5]], [[59.25]], [[47.75]], [[15.5]], [[45.0]], [[15.875]], [[19.375]], [[46.0]], [[40.75]], [[71.0]], [[59.75]], [[14.0]], [[18.5]], [[26.875]], [[48.5]], [[42.25]], [[40.5]], [[7.28125]], [[38.5]], [[32.75]], [[43.25]], [[8.625]], [[5.28125]], [[35.75]], [[35.0]], [[25.5]], [[26.25]], [[26.5]], [[38.5]], [[45.5]], [[39.0]], [[16.375]], [[49.75]], [[26.125]], [[18.5]], [[55.75]], [[79.5]], [[44.0]], [[17.375]], [[23.25]], [[11.4375]], [[57.25]], [[17.375]], [[17.125]], [[44.25]], [[35.75]], [[37.25]], [[29.25]], [[51.25]], [[22.25]], [[7.875]], [[30.0]], [[61.25]]]}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.4375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.875]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.17.feed_forward.shared_expert.down_proj": {"inputs": [[[21.375]]], "outputs": [[[3.03125]], [[3.349654549377988e+38]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.18.self_attn.qkv_proj": {"inputs": [[[22.125]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.18.self_attn.o_proj": {"inputs": [[[3.828125]]], "outputs": [[[6.375]], [[11.5625]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.18.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.96484375]], [[16.875]]]}, "language_model.model.layers.18.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.65625]]]}, "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.96484375]]]}, "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.578125]]]}, "language_model.model.layers.18.self_attn.attn.impl.k_cache": {"inputs": [[[16.875]]]}, "language_model.model.layers.18.self_attn.attn.impl.v_cache": {"inputs": [[[4.65625]]]}, "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.4375]], [[16.875]], [[4.65625]]], "outputs": [[[3.828125]], [[1.0]]]}, "language_model.model.layers.18.feed_forward.gate_up_proj": {"inputs": [[[18.5]]], "params": {"weight": [[0.44140625]]}}, "language_model.model.layers.18.feed_forward.down_proj": {"inputs": [[[18.5]]], "outputs": [[[8.25]], [[358.0]]], "params": {"weight": [[0.490234375]]}}, "language_model.model.layers.19.self_attn.qkv_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.19.self_attn.o_proj": {"inputs": [[[4.15625]]], "outputs": [[[31.625]], [[36.75]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.19.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8515625]], [[11.375]]]}, "language_model.model.layers.19.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.1875]]]}, "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8515625]]]}, "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.4375]]]}, "language_model.model.layers.19.self_attn.attn.impl.k_cache": {"inputs": [[[11.375]]]}, "language_model.model.layers.19.self_attn.attn.impl.v_cache": {"inputs": [[[7.1875]]]}, "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.1875]], [[11.375]], [[7.1875]]], "outputs": [[[4.15625]], [[1.0]]]}, "language_model.model.layers.19.feed_forward.router": {"inputs": [[[23.25]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op": {"inputs": [[[23.25]]], "outputs": [[[4.875]], [[19.0]], [[47.0]], [[17.375]], [[55.5]], [[78.0]], [[47.0]], [[12.9375]], [[50.0]], [[22.75]], [[13.625]], [[7.875]], [[33.0]], [[25.75]], [[23.125]], [[40.75]], [[9.25]], [[9.0]], [[47.75]], [[54.5]], [[22.25]], [[15.375]], [[24.875]], [[40.75]], [[20.375]], [[33.0]], [[61.75]], [[28.25]], [[55.25]], [[55.0]], [[27.0]], [[14.9375]], [[36.75]], [[61.0]], [[19.0]], [[42.5]], [[32.75]], [[12.125]], [[42.0]], [[25.0]], [[10.25]], [[32.0]], [[12.375]], [[31.125]], [[33.0]], [[30.875]], [[19.75]], [[10.375]], [[18.25]], [[32.0]], [[88.0]], [[13.3125]], [[24.0]], [[24.625]], [[27.125]], [[38.0]], [[15.6875]], [[33.25]], [[20.5]], [[64.5]], [[60.5]], [[39.25]], [[59.5]], [[11.9375]], [[29.0]], [[44.5]], [[50.0]], [[60.0]], [[17.375]], [[47.75]], [[7.3125]], [[19.875]], [[33.0]], [[55.75]], [[46.5]], [[16.5]], [[18.875]], [[16.0]], [[38.5]], [[44.5]], [[11.5625]], [[41.25]], [[18.375]], [[28.375]], [[35.75]], [[33.5]], [[47.75]], [[45.75]], [[7.15625]], [[25.375]], [[21.875]], [[21.625]], [[47.0]], [[20.75]], [[31.75]], [[23.0]], [[35.75]], [[25.0]], [[31.625]], [[17.875]], [[10.1875]], [[61.75]], [[21.875]], [[25.125]], [[26.875]], [[31.25]], [[22.875]], [[46.25]], [[24.25]], [[30.75]], [[50.5]], [[16.5]], [[34.25]], [[33.5]], [[38.0]], [[17.375]], [[12.75]], [[27.875]], [[37.5]], [[42.5]], [[36.75]], [[13.0625]], [[12.5625]], [[17.125]], [[29.375]], [[8.8125]], [[10.25]], [[20.5]], [[36.25]]]}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.341796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.3984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.25]]], "params": {"weight": [[0.43359375]]}}, "language_model.model.layers.19.feed_forward.shared_expert.down_proj": {"inputs": [[[19.25]]], "outputs": [[[4.9375]], [[3.349654549377988e+38]]], "params": {"weight": [[0.498046875]]}}, "language_model.model.layers.20.self_attn.qkv_proj": {"inputs": [[[52.5]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.20.self_attn.o_proj": {"inputs": [[[4.90625]]], "outputs": [[[9.5]], [[18.375]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.20.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0625]], [[16.875]]]}, "language_model.model.layers.20.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.25]]]}, "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0625]]]}, "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.90625]]]}, "language_model.model.layers.20.self_attn.attn.impl.k_cache": {"inputs": [[[16.875]]]}, "language_model.model.layers.20.self_attn.attn.impl.v_cache": {"inputs": [[[11.25]]]}, "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.375]], [[16.875]], [[11.25]]], "outputs": [[[4.21875]], [[1.0]]]}, "language_model.model.layers.20.feed_forward.gate_up_proj": {"inputs": [[[41.5]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.20.feed_forward.down_proj": {"inputs": [[[21.125]]], "outputs": [[[9.4375]], [[38.5]]], "params": {"weight": [[0.69140625]]}}, "language_model.model.layers.21.self_attn.qkv_proj": {"inputs": [[[56.25]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.21.self_attn.o_proj": {"inputs": [[[7.5]]], "outputs": [[[25.125]], [[37.75]]], "params": {"weight": [[0.416015625]]}}, "language_model.model.layers.21.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.15625]], [[16.25]]]}, "language_model.model.layers.21.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.8125]]]}, "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.15625]]]}, "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.375]]]}, "language_model.model.layers.21.self_attn.attn.impl.k_cache": {"inputs": [[[16.25]]]}, "language_model.model.layers.21.self_attn.attn.impl.v_cache": {"inputs": [[[8.8125]]]}, "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.4375]], [[16.25]], [[8.8125]]], "outputs": [[[7.5]], [[1.0]]]}, "language_model.model.layers.21.feed_forward.router": {"inputs": [[[24.5]]], "params": {"weight": [[0.45703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op": {"inputs": [[[24.5]]], "outputs": [[[4.53125]], [[14.125]], [[45.75]], [[37.25]], [[47.0]], [[10.75]], [[78.5]], [[16.625]], [[23.875]], [[27.25]], [[31.25]], [[20.375]], [[11.9375]], [[15.1875]], [[49.75]], [[23.125]], [[69.0]], [[76.5]], [[17.125]], [[14.0625]], [[49.75]], [[53.25]], [[27.0]], [[30.5]], [[16.375]], [[51.25]], [[15.6875]], [[13.0625]], [[9.9375]], [[13.5]], [[20.75]], [[13.4375]], [[38.0]], [[52.0]], [[20.25]], [[12.875]], [[12.875]], [[44.0]], [[27.625]], [[44.5]], [[61.5]], [[8.375]], [[27.0]], [[17.125]], [[8.9375]], [[30.75]], [[19.875]], [[10.6875]], [[37.5]], [[37.75]], [[27.625]], [[57.25]], [[25.625]], [[32.5]], [[25.75]], [[13.25]], [[28.75]], [[34.25]], [[13.3125]], [[11.25]], [[49.25]], [[42.25]], [[12.4375]], [[35.5]], [[48.0]], [[20.375]], [[66.0]], [[17.375]], [[28.875]], [[11.375]], [[19.375]], [[14.875]], [[22.625]], [[58.5]], [[33.0]], [[14.5]], [[12.5625]], [[47.75]], [[15.75]], [[28.625]], [[6.59375]], [[36.25]], [[17.0]], [[55.0]], [[34.75]], [[37.5]], [[26.375]], [[9.0]], [[25.625]], [[4.71875]], [[14.25]], [[27.25]], [[23.125]], [[33.5]], [[14.375]], [[17.5]], [[18.0]], [[21.75]], [[25.375]], [[22.375]], [[28.625]], [[10.125]], [[50.25]], [[37.75]], [[69.5]], [[32.75]], [[26.5]], [[31.625]], [[48.5]], [[48.75]], [[24.75]], [[39.0]], [[42.75]], [[19.125]], [[16.125]], [[22.625]], [[19.5]], [[35.0]], [[15.9375]], [[48.25]], [[36.0]], [[10.625]], [[35.5]], [[29.875]], [[26.25]], [[11.75]], [[49.0]], [[16.375]], [[51.5]]]}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.462890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.396484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[24.5]]], "params": {"weight": [[0.41796875]]}}, "language_model.model.layers.21.feed_forward.shared_expert.down_proj": {"inputs": [[[30.125]]], "outputs": [[[4.375]], [[3.349654549377988e+38]]], "params": {"weight": [[0.443359375]]}}, "language_model.model.layers.22.self_attn.qkv_proj": {"inputs": [[[55.75]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.22.self_attn.o_proj": {"inputs": [[[5.1875]]], "outputs": [[[17.25]], [[23.25]]], "params": {"weight": [[0.50390625]]}}, "language_model.model.layers.22.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.28125]], [[14.8125]]]}, "language_model.model.layers.22.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.5]]]}, "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.28125]]]}, "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.5]]]}, "language_model.model.layers.22.self_attn.attn.impl.k_cache": {"inputs": [[[14.8125]]]}, "language_model.model.layers.22.self_attn.attn.impl.v_cache": {"inputs": [[[8.5]]]}, "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[17.75]], [[14.8125]], [[8.5]]], "outputs": [[[5.1875]], [[1.0]]]}, "language_model.model.layers.22.feed_forward.gate_up_proj": {"inputs": [[[44.75]]], "params": {"weight": [[0.625]]}}, "language_model.model.layers.22.feed_forward.down_proj": {"inputs": [[[21.125]]], "outputs": [[[12.0]], [[80.5]]], "params": {"weight": [[0.80859375]]}}, "language_model.model.layers.23.self_attn.qkv_proj": {"inputs": [[[67.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.23.self_attn.o_proj": {"inputs": [[[7.34375]]], "outputs": [[[50.5]], [[61.5]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.23.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0625]], [[12.625]]]}, "language_model.model.layers.23.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.5]]]}, "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0625]]]}, "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.21875]]]}, "language_model.model.layers.23.self_attn.attn.impl.k_cache": {"inputs": [[[12.625]]]}, "language_model.model.layers.23.self_attn.attn.impl.v_cache": {"inputs": [[[11.5]]]}, "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.375]], [[12.625]], [[11.5]]], "outputs": [[[7.34375]], [[1.0]]]}, "language_model.model.layers.23.feed_forward.router": {"inputs": [[[23.625]]], "params": {"weight": [[0.474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op": {"inputs": [[[23.625]]], "outputs": [[[5.75]], [[74.5]], [[7.84375]], [[36.5]], [[59.0]], [[9.875]], [[50.0]], [[22.0]], [[29.625]], [[76.0]], [[17.0]], [[18.5]], [[52.75]], [[31.25]], [[12.5]], [[33.75]], [[45.5]], [[30.375]], [[23.5]], [[22.625]], [[24.875]], [[19.625]], [[89.0]], [[7.875]], [[14.4375]], [[46.75]], [[27.5]], [[44.25]], [[61.75]], [[13.25]], [[13.9375]], [[11.3125]], [[19.5]], [[14.125]], [[16.25]], [[9.3125]], [[41.75]], [[24.625]], [[15.3125]], [[16.375]], [[34.5]], [[38.5]], [[18.25]], [[33.75]], [[10.9375]], [[22.0]], [[7.25]], [[50.25]], [[31.25]], [[67.5]], [[65.0]], [[26.875]], [[52.5]], [[51.25]], [[19.875]], [[39.25]], [[24.0]], [[34.0]], [[12.6875]], [[77.5]], [[29.0]], [[17.0]], [[51.5]], [[14.0]], [[8.75]], [[13.0625]], [[12.8125]], [[48.75]], [[58.25]], [[35.75]], [[53.5]], [[8.6875]], [[100.5]], [[57.75]], [[58.5]], [[13.75]], [[30.25]], [[46.25]], [[45.0]], [[42.75]], [[39.0]], [[22.125]], [[13.0625]], [[64.5]], [[28.25]], [[5.90625]], [[46.75]], [[35.5]], [[8.5625]], [[45.25]], [[13.75]], [[21.5]], [[48.75]], [[40.5]], [[11.375]], [[21.5]], [[41.5]], [[58.5]], [[39.0]], [[14.375]], [[49.25]], [[9.0]], [[18.625]], [[32.5]], [[12.6875]], [[29.625]], [[39.0]], [[14.25]], [[21.5]], [[11.75]], [[91.0]], [[11.0]], [[94.0]], [[10.0]], [[53.25]], [[41.5]], [[8.4375]], [[19.375]], [[58.5]], [[20.125]], [[9.4375]], [[20.0]], [[55.0]], [[42.0]], [[59.25]], [[34.5]], [[26.75]], [[14.6875]], [[8.9375]]]}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.3046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.39453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.625]]], "params": {"weight": [[1.3359375]]}}, "language_model.model.layers.23.feed_forward.shared_expert.down_proj": {"inputs": [[[26.0]]], "outputs": [[[13.8125]], [[3.349654549377988e+38]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.24.self_attn.qkv_proj": {"inputs": [[[60.5]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.24.self_attn.o_proj": {"inputs": [[[4.75]]], "outputs": [[[36.0]], [[42.5]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.24.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.078125]], [[18.0]]]}, "language_model.model.layers.24.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[10.125]]]}, "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.078125]]]}, "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.6875]]]}, "language_model.model.layers.24.self_attn.attn.impl.k_cache": {"inputs": [[[18.0]]]}, "language_model.model.layers.24.self_attn.attn.impl.v_cache": {"inputs": [[[10.125]]]}, "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.8125]], [[18.0]], [[10.125]]], "outputs": [[[4.75]], [[1.0]]]}, "language_model.model.layers.24.feed_forward.gate_up_proj": {"inputs": [[[40.5]]], "params": {"weight": [[0.5859375]]}}, "language_model.model.layers.24.feed_forward.down_proj": {"inputs": [[[30.625]]], "outputs": [[[13.3125]], [[62.5]]], "params": {"weight": [[0.83203125]]}}, "language_model.model.layers.25.self_attn.qkv_proj": {"inputs": [[[49.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.25.self_attn.o_proj": {"inputs": [[[4.6875]]], "outputs": [[[47.75]], [[57.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.25.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.03125]], [[17.875]]]}, "language_model.model.layers.25.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[10.375]]]}, "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.03125]]]}, "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.6875]]]}, "language_model.model.layers.25.self_attn.attn.impl.k_cache": {"inputs": [[[17.875]]]}, "language_model.model.layers.25.self_attn.attn.impl.v_cache": {"inputs": [[[10.375]]]}, "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.1875]], [[17.875]], [[10.375]]], "outputs": [[[4.6875]], [[1.0]]]}, "language_model.model.layers.25.feed_forward.router": {"inputs": [[[30.375]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op": {"inputs": [[[30.375]]], "outputs": [[[6.84375]], [[55.25]], [[49.75]], [[20.125]], [[38.25]], [[35.5]], [[44.75]], [[23.25]], [[24.0]], [[10.375]], [[74.0]], [[9.4375]], [[29.875]], [[47.25]], [[53.5]], [[62.75]], [[28.375]], [[56.0]], [[20.5]], [[50.75]], [[32.75]], [[34.5]], [[63.5]], [[52.75]], [[29.75]], [[8.125]], [[16.75]], [[44.75]], [[22.875]], [[12.875]], [[45.0]], [[16.5]], [[24.875]], [[15.875]], [[107.5]], [[22.375]], [[7.53125]], [[17.875]], [[24.875]], [[13.0]], [[53.5]], [[27.125]], [[37.5]], [[39.0]], [[77.5]], [[24.375]], [[55.25]], [[99.0]], [[16.5]], [[14.3125]], [[37.5]], [[36.5]], [[8.625]], [[18.125]], [[16.125]], [[38.75]], [[25.125]], [[16.0]], [[30.125]], [[10.375]], [[11.3125]], [[34.25]], [[43.75]], [[13.4375]], [[15.125]], [[101.5]], [[10.4375]], [[39.75]], [[34.0]], [[13.9375]], [[36.5]], [[15.5]], [[25.375]], [[31.375]], [[48.0]], [[82.5]], [[45.25]], [[12.1875]], [[20.375]], [[22.125]], [[57.5]], [[14.5]], [[31.0]], [[36.25]], [[53.0]], [[7.0625]], [[13.6875]], [[33.5]], [[42.75]], [[17.0]], [[54.5]], [[12.3125]], [[12.4375]], [[51.25]], [[23.75]], [[8.125]], [[48.25]], [[32.5]], [[45.75]], [[11.75]], [[36.75]], [[61.0]], [[38.25]], [[15.4375]], [[11.8125]], [[20.0]], [[24.875]], [[39.25]], [[5.25]], [[45.25]], [[55.5]], [[71.5]], [[25.5]], [[39.75]], [[42.5]], [[36.25]], [[50.0]], [[80.5]], [[12.25]], [[85.0]], [[66.0]], [[57.5]], [[31.875]], [[35.25]], [[16.375]], [[102.5]], [[7.46875]], [[11.3125]], [[11.1875]]]}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[30.375]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.25.feed_forward.shared_expert.down_proj": {"inputs": [[[27.125]]], "outputs": [[[6.65625]], [[3.349654549377988e+38]]], "params": {"weight": [[0.65625]]}}, "language_model.model.layers.26.self_attn.qkv_proj": {"inputs": [[[63.25]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.26.self_attn.o_proj": {"inputs": [[[6.875]]], "outputs": [[[29.0]], [[53.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.26.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0859375]], [[18.125]]]}, "language_model.model.layers.26.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.6875]]]}, "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0859375]]]}, "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.09375]]]}, "language_model.model.layers.26.self_attn.attn.impl.k_cache": {"inputs": [[[18.125]]]}, "language_model.model.layers.26.self_attn.attn.impl.v_cache": {"inputs": [[[9.6875]]]}, "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.625]], [[18.125]], [[9.6875]]], "outputs": [[[6.875]], [[1.0]]]}, "language_model.model.layers.26.feed_forward.gate_up_proj": {"inputs": [[[45.5]]], "params": {"weight": [[0.365234375]]}}, "language_model.model.layers.26.feed_forward.down_proj": {"inputs": [[[21.0]]], "outputs": [[[15.25]], [[103.0]]], "params": {"weight": [[0.57421875]]}}, "language_model.model.layers.27.self_attn.qkv_proj": {"inputs": [[[72.5]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.27.self_attn.o_proj": {"inputs": [[[7.5]]], "outputs": [[[60.0]], [[66.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.27.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0078125]], [[15.8125]]]}, "language_model.model.layers.27.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[14.0]]]}, "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0078125]]]}, "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.3125]]]}, "language_model.model.layers.27.self_attn.attn.impl.k_cache": {"inputs": [[[15.8125]]]}, "language_model.model.layers.27.self_attn.attn.impl.v_cache": {"inputs": [[[14.0]]]}, "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.375]], [[15.8125]], [[14.0]]], "outputs": [[[7.5]], [[1.0]]]}, "language_model.model.layers.27.feed_forward.router": {"inputs": [[[30.875]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op": {"inputs": [[[30.875]]], "outputs": [[[5.9375]], [[19.125]], [[42.75]], [[51.5]], [[9.0625]], [[15.6875]], [[31.125]], [[56.0]], [[34.75]], [[38.5]], [[71.5]], [[35.25]], [[29.625]], [[64.0]], [[15.8125]], [[66.0]], [[36.75]], [[25.375]], [[57.75]], [[27.0]], [[23.125]], [[31.25]], [[12.5625]], [[28.5]], [[26.375]], [[30.25]], [[11.0]], [[95.5]], [[16.25]], [[29.75]], [[57.0]], [[86.5]], [[15.375]], [[59.5]], [[12.625]], [[66.5]], [[16.625]], [[23.75]], [[51.5]], [[46.0]], [[24.75]], [[25.875]], [[48.25]], [[30.375]], [[28.625]], [[11.125]], [[24.875]], [[13.4375]], [[23.875]], [[44.25]], [[14.125]], [[36.0]], [[8.375]], [[44.25]], [[45.0]], [[30.375]], [[44.0]], [[29.75]], [[28.25]], [[61.25]], [[12.5]], [[24.0]], [[34.75]], [[42.75]], [[24.5]], [[52.75]], [[95.0]], [[14.6875]], [[21.25]], [[26.125]], [[15.125]], [[17.875]], [[27.625]], [[91.0]], [[28.5]], [[24.625]], [[64.0]], [[38.0]], [[36.5]], [[42.75]], [[43.5]], [[61.5]], [[45.0]], [[39.25]], [[69.5]], [[25.5]], [[25.5]], [[20.75]], [[58.75]], [[16.0]], [[19.5]], [[12.0625]], [[52.75]], [[20.125]], [[45.5]], [[23.625]], [[28.75]], [[22.5]], [[38.5]], [[20.375]], [[19.375]], [[56.0]], [[40.5]], [[12.125]], [[42.5]], [[13.375]], [[14.5]], [[14.25]], [[41.5]], [[15.3125]], [[53.0]], [[15.0]], [[46.5]], [[10.75]], [[16.25]], [[43.0]], [[17.875]], [[29.0]], [[31.25]], [[28.75]], [[62.5]], [[12.9375]], [[40.0]], [[22.5]], [[80.5]], [[13.8125]], [[22.0]], [[15.375]], [[26.375]]]}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.38671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.384765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[30.875]]], "params": {"weight": [[0.4140625]]}}, "language_model.model.layers.27.feed_forward.shared_expert.down_proj": {"inputs": [[[68.5]]], "outputs": [[[50.75]], [[2.8744555408848806e+37]]], "params": {"weight": [[0.70703125]]}}, "language_model.model.layers.28.self_attn.qkv_proj": {"inputs": [[[77.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.28.self_attn.o_proj": {"inputs": [[[5.0625]]], "outputs": [[[72.0]], [[74.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.28.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0390625]], [[15.375]]]}, "language_model.model.layers.28.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[10.0625]]]}, "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0390625]]]}, "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.15625]]]}, "language_model.model.layers.28.self_attn.attn.impl.k_cache": {"inputs": [[[15.375]]]}, "language_model.model.layers.28.self_attn.attn.impl.v_cache": {"inputs": [[[10.0625]]]}, "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.9375]], [[15.375]], [[10.0625]]], "outputs": [[[5.0625]], [[1.0]]]}, "language_model.model.layers.28.feed_forward.gate_up_proj": {"inputs": [[[40.25]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.28.feed_forward.down_proj": {"inputs": [[[29.625]]], "outputs": [[[95.0]], [[103.5]]], "params": {"weight": [[0.6953125]]}}, "language_model.model.layers.29.self_attn.qkv_proj": {"inputs": [[[66.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.29.self_attn.o_proj": {"inputs": [[[5.21875]]], "outputs": [[[70.5]], [[76.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.29.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.4140625]], [[17.375]]]}, "language_model.model.layers.29.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.84375]]]}, "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.4140625]]]}, "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.78125]]]}, "language_model.model.layers.29.self_attn.attn.impl.k_cache": {"inputs": [[[17.375]]]}, "language_model.model.layers.29.self_attn.attn.impl.v_cache": {"inputs": [[[6.84375]]]}, "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[16.625]], [[17.25]], [[6.84375]]], "outputs": [[[5.21875]], [[1.0]]]}, "language_model.model.layers.29.feed_forward.router": {"inputs": [[[31.75]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op": {"inputs": [[[31.75]]], "outputs": [[[7.90625]], [[14.0625]], [[42.5]], [[55.0]], [[36.5]], [[29.25]], [[35.5]], [[11.6875]], [[41.25]], [[38.0]], [[32.5]], [[12.6875]], [[16.5]], [[13.625]], [[21.5]], [[30.875]], [[16.875]], [[57.0]], [[51.5]], [[42.0]], [[14.75]], [[31.75]], [[13.75]], [[44.75]], [[10.1875]], [[60.5]], [[21.875]], [[28.125]], [[23.25]], [[14.1875]], [[54.5]], [[34.75]], [[12.6875]], [[41.5]], [[49.5]], [[12.0]], [[54.0]], [[27.375]], [[27.5]], [[42.75]], [[47.0]], [[13.125]], [[22.5]], [[14.875]], [[59.25]], [[29.5]], [[24.875]], [[22.375]], [[27.0]], [[58.5]], [[18.625]], [[17.375]], [[24.125]], [[44.0]], [[88.0]], [[27.0]], [[39.75]], [[20.625]], [[34.0]], [[29.625]], [[11.6875]], [[38.0]], [[11.9375]], [[17.625]], [[20.0]], [[45.25]], [[48.25]], [[30.0]], [[38.25]], [[55.25]], [[48.75]], [[24.875]], [[44.5]], [[28.75]], [[31.375]], [[51.75]], [[51.75]], [[22.375]], [[36.5]], [[38.0]], [[18.625]], [[28.0]], [[13.9375]], [[24.5]], [[35.75]], [[23.5]], [[40.5]], [[30.0]], [[15.0]], [[20.75]], [[22.75]], [[23.125]], [[40.25]], [[36.0]], [[14.125]], [[14.1875]], [[27.5]], [[19.75]], [[14.6875]], [[36.75]], [[20.5]], [[18.375]], [[24.75]], [[21.25]], [[48.0]], [[11.8125]], [[108.0]], [[60.0]], [[28.0]], [[21.25]], [[28.125]], [[23.0]], [[37.25]], [[48.0]], [[11.875]], [[34.5]], [[44.75]], [[22.0]], [[40.0]], [[85.0]], [[26.5]], [[45.5]], [[16.75]], [[49.25]], [[70.0]], [[18.625]], [[20.75]], [[88.5]], [[42.0]]]}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.46484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.5078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[31.75]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.29.feed_forward.shared_expert.down_proj": {"inputs": [[[53.5]]], "outputs": [[[19.125]], [[1.6290831393653022e+35]]], "params": {"weight": [[0.50390625]]}}, "language_model.model.layers.30.self_attn.qkv_proj": {"inputs": [[[71.5]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.30.self_attn.o_proj": {"inputs": [[[5.875]]], "outputs": [[[52.25]], [[57.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.30.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.125]], [[16.375]]]}, "language_model.model.layers.30.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.125]]]}, "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.125]]]}, "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.59375]]]}, "language_model.model.layers.30.self_attn.attn.impl.k_cache": {"inputs": [[[16.375]]]}, "language_model.model.layers.30.self_attn.attn.impl.v_cache": {"inputs": [[[8.125]]]}, "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.0625]], [[16.375]], [[8.125]]], "outputs": [[[5.875]], [[1.0]]]}, "language_model.model.layers.30.feed_forward.gate_up_proj": {"inputs": [[[32.0]]], "params": {"weight": [[0.380859375]]}}, "language_model.model.layers.30.feed_forward.down_proj": {"inputs": [[[30.125]]], "outputs": [[[18.625]], [[139.0]]], "params": {"weight": [[0.6015625]]}}, "language_model.model.layers.31.self_attn.qkv_proj": {"inputs": [[[71.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.31.self_attn.o_proj": {"inputs": [[[6.59375]]], "outputs": [[[66.5]], [[88.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.31.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.140625]], [[11.6875]]]}, "language_model.model.layers.31.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[13.8125]]]}, "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.140625]]]}, "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[8.0625]]]}, "language_model.model.layers.31.self_attn.attn.impl.k_cache": {"inputs": [[[11.6875]]]}, "language_model.model.layers.31.self_attn.attn.impl.v_cache": {"inputs": [[[13.8125]]]}, "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.875]], [[11.6875]], [[13.8125]]], "outputs": [[[6.15625]], [[1.0]]]}, "language_model.model.layers.31.feed_forward.router": {"inputs": [[[33.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op": {"inputs": [[[33.0]]], "outputs": [[[12.0625]], [[14.25]], [[94.5]], [[18.875]], [[29.0]], [[49.0]], [[49.0]], [[51.25]], [[14.8125]], [[39.5]], [[38.0]], [[30.875]], [[12.3125]], [[21.625]], [[12.125]], [[36.25]], [[13.0625]], [[25.375]], [[27.75]], [[54.25]], [[16.75]], [[11.0625]], [[25.0]], [[47.0]], [[91.0]], [[65.5]], [[50.25]], [[43.25]], [[40.25]], [[16.75]], [[13.625]], [[24.0]], [[18.75]], [[40.25]], [[40.25]], [[17.0]], [[13.75]], [[43.75]], [[29.25]], [[26.875]], [[60.5]], [[73.5]], [[44.25]], [[18.125]], [[34.0]], [[93.0]], [[49.25]], [[31.0]], [[29.75]], [[61.5]], [[34.75]], [[41.75]], [[25.625]], [[50.25]], [[33.75]], [[47.25]], [[20.75]], [[13.25]], [[77.5]], [[27.0]], [[21.625]], [[33.75]], [[32.0]], [[32.25]], [[57.0]], [[46.0]], [[34.25]], [[34.75]], [[45.25]], [[42.5]], [[25.0]], [[37.75]], [[52.0]], [[20.125]], [[46.5]], [[17.0]], [[19.625]], [[35.5]], [[56.0]], [[33.75]], [[41.25]], [[20.75]], [[70.0]], [[70.0]], [[120.5]], [[11.9375]], [[38.75]], [[90.5]], [[20.875]], [[60.25]], [[62.0]], [[70.0]], [[31.75]], [[38.5]], [[14.625]], [[53.25]], [[26.875]], [[32.25]], [[11.3125]], [[23.0]], [[31.0]], [[37.0]], [[20.125]], [[47.0]], [[27.125]], [[18.25]], [[8.25]], [[25.75]], [[12.3125]], [[19.5]], [[68.5]], [[63.0]], [[38.75]], [[45.25]], [[29.375]], [[24.875]], [[33.5]], [[49.25]], [[17.375]], [[51.0]], [[51.0]], [[50.0]], [[26.125]], [[11.1875]], [[13.25]], [[98.5]], [[15.8125]], [[18.625]], [[37.75]]]}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.46484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.3046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.439453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[33.0]]], "params": {"weight": [[0.5234375]]}}, "language_model.model.layers.31.feed_forward.shared_expert.down_proj": {"inputs": [[[43.25]]], "outputs": [[[18.5]], [[3.349654549377988e+38]]], "params": {"weight": [[0.734375]]}}, "language_model.model.layers.32.self_attn.qkv_proj": {"inputs": [[[53.25]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.32.self_attn.o_proj": {"inputs": [[[7.9375]]], "outputs": [[[96.0]], [[105.0]]], "params": {"weight": [[0.8046875]]}}, "language_model.model.layers.32.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.03125]], [[13.6875]]]}, "language_model.model.layers.32.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.9375]]]}, "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.03125]]]}, "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.71875]]]}, "language_model.model.layers.32.self_attn.attn.impl.k_cache": {"inputs": [[[13.6875]]]}, "language_model.model.layers.32.self_attn.attn.impl.v_cache": {"inputs": [[[8.9375]]]}, "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.4375]], [[13.6875]], [[8.9375]]], "outputs": [[[7.9375]], [[1.0]]]}, "language_model.model.layers.32.feed_forward.gate_up_proj": {"inputs": [[[32.75]]], "params": {"weight": [[0.7734375]]}}, "language_model.model.layers.32.feed_forward.down_proj": {"inputs": [[[55.5]]], "outputs": [[[21.125]], [[79.5]]], "params": {"weight": [[1.0078125]]}}, "language_model.model.layers.33.self_attn.qkv_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.33.self_attn.o_proj": {"inputs": [[[6.15625]]], "outputs": [[[97.0]], [[118.0]]], "params": {"weight": [[0.44921875]]}}, "language_model.model.layers.33.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.359375]], [[14.5625]]]}, "language_model.model.layers.33.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.25]]]}, "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.359375]]]}, "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.5]]]}, "language_model.model.layers.33.self_attn.attn.impl.k_cache": {"inputs": [[[14.5625]]]}, "language_model.model.layers.33.self_attn.attn.impl.v_cache": {"inputs": [[[8.25]]]}, "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[16.0]], [[14.5625]], [[8.25]]], "outputs": [[[6.15625]], [[1.0]]]}, "language_model.model.layers.33.feed_forward.router": {"inputs": [[[64.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op": {"inputs": [[[64.0]]], "outputs": [[[9.1875]], [[51.5]], [[28.375]], [[34.25]], [[33.75]], [[26.75]], [[74.5]], [[13.9375]], [[65.5]], [[42.75]], [[41.75]], [[23.0]], [[44.0]], [[57.25]], [[34.0]], [[45.5]], [[20.125]], [[57.75]], [[118.0]], [[24.125]], [[17.0]], [[20.0]], [[18.625]], [[23.125]], [[29.375]], [[24.875]], [[35.5]], [[102.0]], [[75.0]], [[161.0]], [[48.0]], [[21.75]], [[32.75]], [[19.625]], [[92.5]], [[46.25]], [[19.0]], [[26.75]], [[22.5]], [[63.0]], [[25.625]], [[17.75]], [[34.75]], [[50.75]], [[104.5]], [[41.0]], [[26.75]], [[37.5]], [[21.625]], [[51.75]], [[99.5]], [[46.75]], [[18.375]], [[57.75]], [[25.25]], [[134.0]], [[77.5]], [[44.5]], [[18.625]], [[38.25]], [[80.5]], [[44.5]], [[42.5]], [[40.5]], [[88.0]], [[16.375]], [[59.0]], [[68.0]], [[17.875]], [[41.75]], [[44.5]], [[85.5]], [[62.0]], [[71.5]], [[95.0]], [[21.25]], [[85.0]], [[49.25]], [[25.75]], [[31.5]], [[42.25]], [[28.625]], [[18.625]], [[60.75]], [[22.375]], [[16.375]], [[55.5]], [[52.5]], [[57.5]], [[71.5]], [[60.5]], [[21.125]], [[21.75]], [[20.625]], [[46.25]], [[45.0]], [[61.25]], [[24.625]], [[68.5]], [[53.25]], [[61.5]], [[66.0]], [[69.5]], [[45.25]], [[41.25]], [[81.5]], [[43.75]], [[54.75]], [[30.375]], [[19.875]], [[20.0]], [[35.75]], [[134.0]], [[18.375]], [[39.75]], [[28.625]], [[20.25]], [[19.0]], [[52.5]], [[26.75]], [[72.5]], [[25.75]], [[31.25]], [[20.875]], [[11.375]], [[43.5]], [[21.375]], [[17.125]], [[34.0]]]}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.384765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[64.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.33.feed_forward.shared_expert.down_proj": {"inputs": [[[76.5]]], "outputs": [[[11.6875]], [[2.8179633510640217e+38]]], "params": {"weight": [[0.490234375]]}}, "language_model.model.layers.34.self_attn.qkv_proj": {"inputs": [[[54.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.34.self_attn.o_proj": {"inputs": [[[5.09375]]], "outputs": [[[46.75]], [[80.5]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.34.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.328125]], [[14.5]]]}, "language_model.model.layers.34.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.09375]]]}, "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.328125]]]}, "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.28125]]]}, "language_model.model.layers.34.self_attn.attn.impl.k_cache": {"inputs": [[[14.5]]]}, "language_model.model.layers.34.self_attn.attn.impl.v_cache": {"inputs": [[[6.09375]]]}, "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[17.75]], [[14.5]], [[6.09375]]], "outputs": [[[5.09375]], [[1.0]]]}, "language_model.model.layers.34.feed_forward.gate_up_proj": {"inputs": [[[31.125]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.34.feed_forward.down_proj": {"inputs": [[[77.0]]], "outputs": [[[30.375]], [[128.0]]], "params": {"weight": [[0.6875]]}}, "language_model.model.layers.35.self_attn.qkv_proj": {"inputs": [[[52.75]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.35.self_attn.o_proj": {"inputs": [[[7.875]]], "outputs": [[[87.5]], [[93.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.35.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.91796875]], [[12.5]]]}, "language_model.model.layers.35.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.9375]]]}, "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.91796875]]]}, "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.875]]]}, "language_model.model.layers.35.self_attn.attn.impl.k_cache": {"inputs": [[[12.5]]]}, "language_model.model.layers.35.self_attn.attn.impl.v_cache": {"inputs": [[[8.9375]]]}, "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.1875]], [[12.5]], [[8.9375]]], "outputs": [[[6.75]], [[1.0]]]}, "language_model.model.layers.35.feed_forward.router": {"inputs": [[[64.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op": {"inputs": [[[64.0]]], "outputs": [[[16.0]], [[120.0]], [[26.25]], [[16.875]], [[39.5]], [[137.0]], [[34.75]], [[59.75]], [[132.0]], [[117.5]], [[46.75]], [[96.0]], [[62.25]], [[120.5]], [[66.5]], [[82.0]], [[63.25]], [[70.0]], [[29.375]], [[93.0]], [[44.0]], [[33.0]], [[26.375]], [[49.25]], [[30.25]], [[32.5]], [[66.0]], [[104.0]], [[62.25]], [[23.875]], [[64.5]], [[90.0]], [[54.5]], [[66.0]], [[39.75]], [[27.0]], [[96.5]], [[73.5]], [[50.75]], [[71.0]], [[50.0]], [[51.75]], [[48.0]], [[25.875]], [[57.5]], [[103.0]], [[34.5]], [[15.1875]], [[23.125]], [[91.0]], [[85.5]], [[45.0]], [[59.25]], [[50.0]], [[55.0]], [[44.5]], [[38.0]], [[85.5]], [[28.5]], [[74.0]], [[73.0]], [[115.0]], [[23.375]], [[78.5]], [[47.25]], [[55.25]], [[22.25]], [[102.5]], [[110.5]], [[19.375]], [[88.5]], [[114.5]], [[95.0]], [[28.125]], [[33.5]], [[60.25]], [[53.75]], [[49.25]], [[65.5]], [[76.0]], [[92.5]], [[108.5]], [[155.0]], [[34.0]], [[34.75]], [[102.5]], [[132.0]], [[40.25]], [[49.0]], [[18.5]], [[50.0]], [[120.5]], [[113.5]], [[72.5]], [[120.0]], [[72.0]], [[86.0]], [[76.0]], [[37.5]], [[104.5]], [[65.5]], [[150.0]], [[18.125]], [[28.625]], [[61.0]], [[25.125]], [[73.0]], [[43.0]], [[34.75]], [[113.0]], [[63.75]], [[45.5]], [[38.25]], [[46.25]], [[53.5]], [[49.0]], [[68.5]], [[45.5]], [[19.625]], [[55.0]], [[51.25]], [[79.5]], [[25.5]], [[28.5]], [[91.0]], [[56.5]], [[48.25]], [[18.375]], [[67.0]]]}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.69140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.47265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.341796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[64.0]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.35.feed_forward.shared_expert.down_proj": {"inputs": [[[78.5]]], "outputs": [[[10.625]], [[3.017347550431759e+38]]], "params": {"weight": [[0.404296875]]}}, "language_model.model.layers.36.self_attn.qkv_proj": {"inputs": [[[56.75]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.36.self_attn.o_proj": {"inputs": [[[5.03125]]], "outputs": [[[27.75]], [[43.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.36.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.4921875]], [[12.6875]]]}, "language_model.model.layers.36.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.125]]]}, "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.4921875]]]}, "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.59375]]]}, "language_model.model.layers.36.self_attn.attn.impl.k_cache": {"inputs": [[[12.6875]]]}, "language_model.model.layers.36.self_attn.attn.impl.v_cache": {"inputs": [[[6.125]]]}, "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[20.625]], [[12.6875]], [[6.125]]], "outputs": [[[5.03125]], [[1.0]]]}, "language_model.model.layers.36.feed_forward.gate_up_proj": {"inputs": [[[29.625]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.36.feed_forward.down_proj": {"inputs": [[[71.0]]], "outputs": [[[38.25]], [[76.5]]], "params": {"weight": [[0.73828125]]}}, "language_model.model.layers.37.self_attn.qkv_proj": {"inputs": [[[50.25]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.37.self_attn.o_proj": {"inputs": [[[12.375]]], "outputs": [[[102.0]], [[105.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.37.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.953125]], [[13.6875]]]}, "language_model.model.layers.37.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[16.75]]]}, "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.953125]]]}, "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[9.25]]]}, "language_model.model.layers.37.self_attn.attn.impl.k_cache": {"inputs": [[[13.6875]]]}, "language_model.model.layers.37.self_attn.attn.impl.v_cache": {"inputs": [[[16.75]]]}, "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[25.125]], [[13.6875]], [[16.75]]], "outputs": [[[12.375]], [[1.0]]]}, "language_model.model.layers.37.feed_forward.router": {"inputs": [[[70.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op": {"inputs": [[[70.0]]], "outputs": [[[11.9375]], [[70.5]], [[90.0]], [[39.5]], [[49.75]], [[51.5]], [[46.5]], [[58.75]], [[78.5]], [[29.75]], [[62.0]], [[31.0]], [[39.75]], [[65.0]], [[51.5]], [[47.75]], [[54.5]], [[38.25]], [[47.75]], [[86.5]], [[56.75]], [[67.5]], [[110.0]], [[71.5]], [[29.75]], [[70.5]], [[63.75]], [[32.75]], [[96.0]], [[162.0]], [[62.75]], [[26.0]], [[93.5]], [[34.25]], [[44.75]], [[60.0]], [[46.25]], [[102.5]], [[108.5]], [[105.5]], [[65.0]], [[75.0]], [[29.0]], [[26.25]], [[70.5]], [[90.5]], [[107.5]], [[177.0]], [[27.5]], [[76.5]], [[17.5]], [[69.0]], [[52.75]], [[129.0]], [[59.75]], [[42.0]], [[139.0]], [[94.0]], [[32.25]], [[74.0]], [[122.5]], [[97.0]], [[37.5]], [[55.5]], [[30.5]], [[85.0]], [[27.5]], [[49.25]], [[79.5]], [[66.0]], [[42.5]], [[59.5]], [[52.5]], [[51.5]], [[22.5]], [[29.625]], [[66.0]], [[130.0]], [[18.625]], [[55.0]], [[77.5]], [[108.5]], [[27.375]], [[108.5]], [[29.5]], [[64.5]], [[93.5]], [[87.0]], [[125.5]], [[114.0]], [[95.0]], [[28.125]], [[74.5]], [[47.0]], [[49.0]], [[29.375]], [[81.5]], [[22.75]], [[27.5]], [[91.0]], [[92.0]], [[167.0]], [[79.5]], [[54.5]], [[36.5]], [[72.0]], [[38.0]], [[77.0]], [[23.25]], [[86.0]], [[40.0]], [[67.5]], [[62.75]], [[63.25]], [[68.0]], [[43.5]], [[88.0]], [[121.0]], [[106.0]], [[36.5]], [[50.25]], [[40.0]], [[73.0]], [[64.5]], [[47.0]], [[105.0]], [[27.125]], [[74.5]], [[63.0]]]}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.404296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[70.0]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.37.feed_forward.shared_expert.down_proj": {"inputs": [[[103.5]]], "outputs": [[[17.5]], [[3.349654549377988e+38]]], "params": {"weight": [[0.4765625]]}}, "language_model.model.layers.38.self_attn.qkv_proj": {"inputs": [[[47.25]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.38.self_attn.o_proj": {"inputs": [[[9.0]]], "outputs": [[[26.75]], [[56.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.38.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.9453125]], [[13.8125]]]}, "language_model.model.layers.38.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.9375]]]}, "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.9453125]]]}, "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[8.0]]]}, "language_model.model.layers.38.self_attn.attn.impl.k_cache": {"inputs": [[[14.0]]]}, "language_model.model.layers.38.self_attn.attn.impl.v_cache": {"inputs": [[[11.9375]]]}, "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[23.875]], [[14.0]], [[11.9375]]], "outputs": [[[9.0]], [[1.0]]]}, "language_model.model.layers.38.feed_forward.gate_up_proj": {"inputs": [[[31.125]]], "params": {"weight": [[0.435546875]]}}, "language_model.model.layers.38.feed_forward.down_proj": {"inputs": [[[74.5]]], "outputs": [[[24.75]], [[193.0]]], "params": {"weight": [[0.70703125]]}}, "language_model.model.layers.39.self_attn.qkv_proj": {"inputs": [[[49.75]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.39.self_attn.o_proj": {"inputs": [[[9.0]]], "outputs": [[[129.0]], [[134.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.39.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0234375]], [[13.125]]]}, "language_model.model.layers.39.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.3125]]]}, "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0234375]]]}, "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.65625]]]}, "language_model.model.layers.39.self_attn.attn.impl.k_cache": {"inputs": [[[13.125]]]}, "language_model.model.layers.39.self_attn.attn.impl.v_cache": {"inputs": [[[11.3125]]]}, "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.0625]], [[13.125]], [[11.3125]]], "outputs": [[[9.0]], [[1.0]]]}, "language_model.model.layers.39.feed_forward.router": {"inputs": [[[61.5]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op": {"inputs": [[[61.5]]], "outputs": [[[11.75]], [[44.5]], [[33.25]], [[90.5]], [[65.5]], [[77.5]], [[39.25]], [[41.25]], [[60.0]], [[43.75]], [[66.0]], [[75.5]], [[56.0]], [[114.0]], [[121.0]], [[85.5]], [[43.75]], [[131.0]], [[109.5]], [[52.5]], [[62.25]], [[50.25]], [[90.0]], [[36.5]], [[101.5]], [[60.5]], [[117.5]], [[74.5]], [[123.5]], [[73.0]], [[60.75]], [[54.25]], [[111.0]], [[42.5]], [[48.5]], [[97.5]], [[59.25]], [[72.0]], [[74.5]], [[100.0]], [[122.5]], [[66.0]], [[44.5]], [[76.0]], [[39.75]], [[64.5]], [[30.5]], [[59.5]], [[52.25]], [[63.25]], [[100.5]], [[85.5]], [[69.0]], [[67.5]], [[56.75]], [[148.0]], [[89.5]], [[36.0]], [[93.5]], [[61.25]], [[44.0]], [[121.0]], [[109.0]], [[116.0]], [[29.0]], [[30.625]], [[60.0]], [[40.0]], [[83.0]], [[80.0]], [[104.5]], [[137.0]], [[33.0]], [[64.5]], [[116.0]], [[59.0]], [[69.0]], [[115.0]], [[133.0]], [[106.5]], [[46.5]], [[56.75]], [[53.0]], [[62.25]], [[52.25]], [[82.5]], [[93.5]], [[162.0]], [[42.5]], [[100.5]], [[54.0]], [[56.25]], [[60.0]], [[48.25]], [[59.25]], [[45.0]], [[94.5]], [[45.5]], [[112.0]], [[48.0]], [[54.25]], [[62.5]], [[70.5]], [[81.0]], [[105.5]], [[45.0]], [[102.0]], [[81.0]], [[98.0]], [[59.0]], [[49.5]], [[40.25]], [[94.5]], [[22.875]], [[57.75]], [[55.5]], [[224.0]], [[30.5]], [[30.0]], [[100.0]], [[68.0]], [[61.5]], [[69.0]], [[96.5]], [[38.5]], [[103.5]], [[34.0]], [[99.5]], [[103.5]]]}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[61.5]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.39.feed_forward.shared_expert.down_proj": {"inputs": [[[111.0]]], "outputs": [[[63.5]], [[3.349654549377988e+38]]], "params": {"weight": [[0.84375]]}}, "language_model.model.layers.40.self_attn.qkv_proj": {"inputs": [[[46.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.40.self_attn.o_proj": {"inputs": [[[12.5]]], "outputs": [[[93.0]], [[90.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.40.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.625]], [[11.9375]]]}, "language_model.model.layers.40.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[14.25]]]}, "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.625]]]}, "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[12.5]]]}, "language_model.model.layers.40.self_attn.attn.impl.k_cache": {"inputs": [[[11.9375]]]}, "language_model.model.layers.40.self_attn.attn.impl.v_cache": {"inputs": [[[14.25]]]}, "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[21.125]], [[11.9375]], [[14.0625]]], "outputs": [[[11.8125]], [[1.0]]]}, "language_model.model.layers.40.feed_forward.gate_up_proj": {"inputs": [[[40.75]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.40.feed_forward.down_proj": {"inputs": [[[215.0]]], "outputs": [[[22.625]], [[146.0]]], "params": {"weight": [[0.50390625]]}}, "language_model.model.layers.41.self_attn.qkv_proj": {"inputs": [[[33.75]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.41.self_attn.o_proj": {"inputs": [[[13.4375]]], "outputs": [[[159.0]], [[164.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.41.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.265625]], [[11.3125]]]}, "language_model.model.layers.41.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[18.125]]]}, "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.265625]]]}, "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[12.3125]]]}, "language_model.model.layers.41.self_attn.attn.impl.k_cache": {"inputs": [[[11.3125]]]}, "language_model.model.layers.41.self_attn.attn.impl.v_cache": {"inputs": [[[18.125]]]}, "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[16.75]], [[11.3125]], [[18.125]]], "outputs": [[[13.4375]], [[1.0]]]}, "language_model.model.layers.41.feed_forward.router": {"inputs": [[[68.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op": {"inputs": [[[68.0]]], "outputs": [[[20.0]], [[113.0]], [[37.75]], [[34.25]], [[104.0]], [[92.0]], [[44.75]], [[103.0]], [[105.5]], [[36.75]], [[88.0]], [[96.0]], [[46.0]], [[92.5]], [[114.5]], [[115.0]], [[135.0]], [[104.5]], [[86.5]], [[107.0]], [[43.75]], [[87.0]], [[44.75]], [[101.5]], [[119.5]], [[130.0]], [[145.0]], [[133.0]], [[52.75]], [[61.5]], [[72.5]], [[98.5]], [[60.5]], [[143.0]], [[88.0]], [[119.5]], [[98.5]], [[82.0]], [[98.5]], [[53.75]], [[46.75]], [[64.0]], [[138.0]], [[137.0]], [[47.5]], [[73.5]], [[72.0]], [[80.5]], [[59.75]], [[222.0]], [[140.0]], [[94.0]], [[94.5]], [[90.5]], [[111.0]], [[112.0]], [[60.75]], [[162.0]], [[74.0]], [[66.0]], [[113.5]], [[164.0]], [[85.0]], [[69.0]], [[99.0]], [[117.5]], [[54.75]], [[98.0]], [[72.0]], [[29.375]], [[81.0]], [[65.0]], [[29.125]], [[64.5]], [[27.375]], [[73.0]], [[78.5]], [[87.0]], [[67.5]], [[43.75]], [[101.5]], [[139.0]], [[99.0]], [[81.5]], [[65.0]], [[42.5]], [[50.5]], [[68.5]], [[40.0]], [[57.5]], [[89.5]], [[85.0]], [[60.5]], [[121.0]], [[73.5]], [[44.0]], [[245.0]], [[71.5]], [[92.5]], [[70.0]], [[65.0]], [[111.0]], [[91.5]], [[48.75]], [[52.75]], [[114.0]], [[52.75]], [[87.0]], [[158.0]], [[33.75]], [[83.5]], [[159.0]], [[120.0]], [[88.0]], [[55.75]], [[65.5]], [[143.0]], [[81.5]], [[136.0]], [[125.5]], [[66.5]], [[88.5]], [[73.5]], [[111.0]], [[37.0]], [[51.25]], [[78.0]], [[62.25]], [[132.0]]]}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.388671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[68.0]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.41.feed_forward.shared_expert.down_proj": {"inputs": [[[298.0]]], "outputs": [[[57.0]], [[3.349654549377988e+38]]], "params": {"weight": [[0.83203125]]}}, "language_model.model.layers.42.self_attn.qkv_proj": {"inputs": [[[38.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.42.self_attn.o_proj": {"inputs": [[[11.4375]]], "outputs": [[[50.5]], [[134.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.42.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.2890625]], [[10.3125]]]}, "language_model.model.layers.42.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[18.875]]]}, "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.2890625]]]}, "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[11.875]]]}, "language_model.model.layers.42.self_attn.attn.impl.k_cache": {"inputs": [[[10.3125]]]}, "language_model.model.layers.42.self_attn.attn.impl.v_cache": {"inputs": [[[18.875]]]}, "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.6875]], [[10.3125]], [[18.875]]], "outputs": [[[11.4375]], [[1.0]]]}, "language_model.model.layers.42.feed_forward.gate_up_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.42.feed_forward.down_proj": {"inputs": [[[182.0]]], "outputs": [[[59.5]], [[338.0]]], "params": {"weight": [[0.66015625]]}}, "language_model.model.layers.43.self_attn.qkv_proj": {"inputs": [[[45.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.43.self_attn.o_proj": {"inputs": [[[8.9375]]], "outputs": [[[94.5]], [[232.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.43.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.91015625]], [[21.625]]]}, "language_model.model.layers.43.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.625]]]}, "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.91015625]]]}, "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.8125]]]}, "language_model.model.layers.43.self_attn.attn.impl.k_cache": {"inputs": [[[21.625]]]}, "language_model.model.layers.43.self_attn.attn.impl.v_cache": {"inputs": [[[9.625]]]}, "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.4375]], [[21.625]], [[9.625]]], "outputs": [[[8.9375]], [[1.0]]]}, "language_model.model.layers.43.feed_forward.router": {"inputs": [[[68.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op": {"inputs": [[[68.0]]], "outputs": [[[35.0]], [[86.0]], [[79.0]], [[62.0]], [[61.0]], [[153.0]], [[56.75]], [[48.5]], [[116.0]], [[125.0]], [[125.5]], [[83.5]], [[113.5]], [[49.5]], [[86.0]], [[78.5]], [[114.0]], [[115.5]], [[62.75]], [[232.0]], [[38.25]], [[143.0]], [[84.5]], [[180.0]], [[125.0]], [[99.5]], [[76.0]], [[82.5]], [[108.5]], [[99.5]], [[94.5]], [[84.5]], [[56.75]], [[79.0]], [[128.0]], [[99.5]], [[141.0]], [[105.0]], [[56.75]], [[86.0]], [[109.0]], [[74.5]], [[123.5]], [[157.0]], [[44.75]], [[187.0]], [[94.5]], [[54.25]], [[36.0]], [[83.0]], [[80.0]], [[71.5]], [[209.0]], [[182.0]], [[49.0]], [[71.0]], [[130.0]], [[39.5]], [[30.375]], [[96.0]], [[113.5]], [[36.5]], [[67.5]], [[46.5]], [[142.0]], [[179.0]], [[114.5]], [[104.5]], [[57.5]], [[262.0]], [[48.5]], [[89.5]], [[110.0]], [[63.0]], [[91.5]], [[51.25]], [[87.0]], [[151.0]], [[61.5]], [[111.0]], [[148.0]], [[94.5]], [[103.0]], [[119.5]], [[73.5]], [[119.0]], [[103.0]], [[54.25]], [[81.0]], [[93.5]], [[80.0]], [[145.0]], [[85.0]], [[82.0]], [[90.0]], [[120.0]], [[46.75]], [[43.0]], [[123.0]], [[48.25]], [[93.0]], [[91.0]], [[52.75]], [[136.0]], [[109.5]], [[115.0]], [[131.0]], [[95.5]], [[105.5]], [[63.5]], [[64.0]], [[147.0]], [[97.5]], [[117.5]], [[88.5]], [[40.0]], [[67.5]], [[98.0]], [[72.0]], [[77.5]], [[49.75]], [[46.5]], [[130.0]], [[62.25]], [[83.5]], [[75.0]], [[164.0]], [[109.5]], [[84.0]]]}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.5703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.41796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.3046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[68.0]]], "params": {"weight": [[0.359375]]}}, "language_model.model.layers.43.feed_forward.shared_expert.down_proj": {"inputs": [[[243.0]]], "outputs": [[[28.125]], [[3.017347550431759e+38]]], "params": {"weight": [[0.61328125]]}}, "language_model.model.layers.44.self_attn.qkv_proj": {"inputs": [[[28.625]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.44.self_attn.o_proj": {"inputs": [[[17.5]]], "outputs": [[[68.0]], [[153.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.44.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.765625]], [[16.375]]]}, "language_model.model.layers.44.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[18.625]]]}, "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.765625]]]}, "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[17.625]]]}, "language_model.model.layers.44.self_attn.attn.impl.k_cache": {"inputs": [[[16.375]]]}, "language_model.model.layers.44.self_attn.attn.impl.v_cache": {"inputs": [[[18.625]]]}, "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[23.75]], [[16.375]], [[18.625]]], "outputs": [[[17.5]], [[1.0]]]}, "language_model.model.layers.44.feed_forward.gate_up_proj": {"inputs": [[[41.5]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.44.feed_forward.down_proj": {"inputs": [[[189.0]]], "outputs": [[[63.25]], [[191.0]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.45.self_attn.qkv_proj": {"inputs": [[[27.375]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.45.self_attn.o_proj": {"inputs": [[[10.0]]], "outputs": [[[169.0]], [[220.0]]], "params": {"weight": [[0.41796875]]}}, "language_model.model.layers.45.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.65625]], [[14.625]]]}, "language_model.model.layers.45.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[10.8125]]]}, "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.65625]]]}, "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[8.625]]]}, "language_model.model.layers.45.self_attn.attn.impl.k_cache": {"inputs": [[[14.625]]]}, "language_model.model.layers.45.self_attn.attn.impl.v_cache": {"inputs": [[[10.8125]]]}, "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[20.625]], [[14.625]], [[10.8125]]], "outputs": [[[10.0]], [[1.0]]]}, "language_model.model.layers.45.feed_forward.router": {"inputs": [[[51.75]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op": {"inputs": [[[51.75]]], "outputs": [[[14.0]], [[24.5]], [[25.75]], [[52.25]], [[75.5]], [[0.0]], [[57.25]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[54.5]], [[30.25]], [[56.25]], [[38.75]], [[0.0]], [[0.0]], [[0.0]], [[23.75]], [[0.0]], [[0.0]], [[0.0]], [[33.75]], [[0.0]], [[31.25]], [[30.875]], [[26.125]], [[47.25]], [[67.0]], [[0.0]], [[17.75]], [[30.875]], [[49.5]], [[38.0]], [[67.0]], [[25.75]], [[0.0]], [[30.5]], [[21.0]], [[47.25]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[46.25]], [[0.0]], [[196.0]], [[0.0]], [[0.0]], [[0.0]], [[134.0]], [[0.0]], [[97.5]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[38.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[102.5]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[45.25]], [[37.5]], [[115.0]], [[31.125]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[103.5]], [[0.0]], [[58.75]], [[30.75]], [[47.0]], [[69.5]], [[0.0]], [[0.0]], [[19.625]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[19.375]], [[8.782037597132586e-18]], [[46.5]], [[0.0]], [[56.5]], [[0.0]], [[0.0]]]}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.59765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.57421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[51.75]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.45.feed_forward.shared_expert.down_proj": {"inputs": [[[182.0]]], "outputs": [[[28.0]], [[3.349654549377988e+38]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.46.self_attn.qkv_proj": {"inputs": [[[26.75]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.46.self_attn.o_proj": {"inputs": [[[27.375]]], "outputs": [[[101.0]], [[188.0]]], "params": {"weight": [[0.875]]}}, "language_model.model.layers.46.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.8359375]], [[18.875]]]}, "language_model.model.layers.46.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[27.375]]]}, "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.8359375]]]}, "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[27.375]]]}, "language_model.model.layers.46.self_attn.attn.impl.k_cache": {"inputs": [[[18.875]]]}, "language_model.model.layers.46.self_attn.attn.impl.v_cache": {"inputs": [[[27.375]]]}, "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[22.625]], [[18.875]], [[24.5]]], "outputs": [[[24.5]], [[1.0]]]}, "language_model.model.layers.46.feed_forward.gate_up_proj": {"inputs": [[[40.5]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.46.feed_forward.down_proj": {"inputs": [[[358.0]]], "outputs": [[[1688.0]], [[2400.0]]], "params": {"weight": [[0.5078125]]}}, "language_model.model.layers.47.self_attn.qkv_proj": {"inputs": [[[19.875]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.47.self_attn.o_proj": {"inputs": [[[15.3125]]], "outputs": [[[98.0]], [[126.5]]], "params": {"weight": [[0.50390625]]}}, "language_model.model.layers.47.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.6875]], [[7.28125]]]}, "language_model.model.layers.47.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[15.6875]]]}, "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.6875]]]}, "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[14.9375]]]}, "language_model.model.layers.47.self_attn.attn.impl.k_cache": {"inputs": [[[7.28125]]]}, "language_model.model.layers.47.self_attn.attn.impl.v_cache": {"inputs": [[[15.6875]]]}, "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.25]], [[7.28125]], [[15.6875]]], "outputs": [[[15.3125]], [[1.0]]]}, "language_model.model.layers.47.feed_forward.router": {"inputs": [[[33.5]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op": {"inputs": [[[33.5]]], "outputs": [[[584.0]], [[97.5]], [[54.25]], [[68.0]], [[118.0]], [[66.5]], [[30.25]], [[57.25]], [[50.0]], [[93.5]], [[200.0]], [[134.0]], [[82.0]], [[94.0]], [[31.5]], [[80.0]], [[25.5]], [[103.5]], [[51.5]], [[63.25]], [[59.25]], [[93.0]], [[56.5]], [[49.25]], [[45.25]], [[210.0]], [[181.0]], [[28.5]], [[119.0]], [[44.75]], [[99.0]], [[93.0]], [[57.25]], [[27.125]], [[41.5]], [[182.0]], [[52.25]], [[44.0]], [[22.5]], [[197.0]], [[71.0]], [[66.0]], [[94.5]], [[185.0]], [[38.5]], [[40.0]], [[132.0]], [[84.0]], [[45.75]], [[23.75]], [[85.0]], [[112.0]], [[218.0]], [[57.75]], [[24.0]], [[39.75]], [[36.0]], [[77.5]], [[55.5]], [[32.0]], [[108.5]], [[59.25]], [[107.5]], [[528.0]], [[114.5]], [[52.75]], [[37.75]], [[54.0]], [[156.0]], [[36.5]], [[138.0]], [[123.0]], [[28.625]], [[36.0]], [[24.125]], [[80.5]], [[39.5]], [[34.5]], [[40.25]], [[120.5]], [[70.5]], [[73.0]], [[292.0]], [[430.0]], [[53.0]], [[45.0]], [[88.0]], [[88.0]], [[59.0]], [[53.0]], [[95.0]], [[237.0]], [[125.5]], [[122.5]], [[30.125]], [[57.25]], [[81.0]], [[77.5]], [[39.5]], [[67.0]], [[94.0]], [[59.5]], [[50.25]], [[83.5]], [[42.25]], [[109.5]], [[33.0]], [[48.75]], [[266.0]], [[53.25]], [[81.0]], [[83.0]], [[151.0]], [[38.75]], [[41.25]], [[111.5]], [[83.0]], [[47.5]], [[143.0]], [[33.5]], [[268.0]], [[145.0]], [[64.5]], [[82.5]], [[28.5]], [[59.0]], [[123.5]], [[122.0]], [[249.0]]]}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.76171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.392578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.59375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.7109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.38671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.65234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.3671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.84375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.5546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.37109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.51171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.416015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.6015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.384765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.78515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.46875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.49609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.55078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.6484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.6875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.8125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.427734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.6796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.7265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.439453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.431640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.478515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.58203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.79296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.4453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.46875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.5546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.48046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.58203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.76953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.75]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.6796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.6640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.44140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.76953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.57421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.53515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.59765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.61328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.43359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.48828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.68359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.5859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.69921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.46875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.62109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.609375]]}}, "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[33.5]]], "params": {"weight": [[0.38671875]]}}, "language_model.model.layers.47.feed_forward.shared_expert.down_proj": {"inputs": [[[804.0]]], "outputs": [[[450.0]], [[3.017347550431759e+38]]], "params": {"weight": [[0.625]]}}, "language_model.lm_head": {"inputs": [[[52.0]]], "params": {"weight": [[0.8203125]]}}}} \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_0_8.npz b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_0_8.npz new file mode 100644 index 000000000000..7173bd2f4e4a Binary files /dev/null and b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_0_8.npz differ diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_0_8_mod_list.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_0_8_mod_list.json new file mode 100644 index 000000000000..928cd9e2a2c5 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_0_8_mod_list.json @@ -0,0 +1,6935 @@ +[ + "vision_model.patch_embedding.linear", + "vision_model.model.layers.0.self_attn.qkv_proj", + "vision_model.model.layers.0.self_attn.o_proj", + "vision_model.model.layers.0.mlp.fc1", + "vision_model.model.layers.0.mlp.fc2", + "vision_model.model.layers.1.self_attn.qkv_proj", + "vision_model.model.layers.1.self_attn.o_proj", + "vision_model.model.layers.1.mlp.fc1", + "vision_model.model.layers.1.mlp.fc2", + "vision_model.model.layers.2.self_attn.qkv_proj", + "vision_model.model.layers.2.self_attn.o_proj", + "vision_model.model.layers.2.mlp.fc1", + "vision_model.model.layers.2.mlp.fc2", + "vision_model.model.layers.3.self_attn.qkv_proj", + "vision_model.model.layers.3.self_attn.o_proj", + "vision_model.model.layers.3.mlp.fc1", + "vision_model.model.layers.3.mlp.fc2", + "vision_model.model.layers.4.self_attn.qkv_proj", + "vision_model.model.layers.4.self_attn.o_proj", + "vision_model.model.layers.4.mlp.fc1", + "vision_model.model.layers.4.mlp.fc2", + "vision_model.model.layers.5.self_attn.qkv_proj", + "vision_model.model.layers.5.self_attn.o_proj", + "vision_model.model.layers.5.mlp.fc1", + "vision_model.model.layers.5.mlp.fc2", + "vision_model.model.layers.6.self_attn.qkv_proj", + "vision_model.model.layers.6.self_attn.o_proj", + "vision_model.model.layers.6.mlp.fc1", + "vision_model.model.layers.6.mlp.fc2", + "vision_model.model.layers.7.self_attn.qkv_proj", + "vision_model.model.layers.7.self_attn.o_proj", + "vision_model.model.layers.7.mlp.fc1", + "vision_model.model.layers.7.mlp.fc2", + "vision_model.model.layers.8.self_attn.qkv_proj", + "vision_model.model.layers.8.self_attn.o_proj", + "vision_model.model.layers.8.mlp.fc1", + "vision_model.model.layers.8.mlp.fc2", + "vision_model.model.layers.9.self_attn.qkv_proj", + "vision_model.model.layers.9.self_attn.o_proj", + "vision_model.model.layers.9.mlp.fc1", + "vision_model.model.layers.9.mlp.fc2", + "vision_model.model.layers.10.self_attn.qkv_proj", + "vision_model.model.layers.10.self_attn.o_proj", + "vision_model.model.layers.10.mlp.fc1", + "vision_model.model.layers.10.mlp.fc2", + "vision_model.model.layers.11.self_attn.qkv_proj", + "vision_model.model.layers.11.self_attn.o_proj", + "vision_model.model.layers.11.mlp.fc1", + "vision_model.model.layers.11.mlp.fc2", + "vision_model.model.layers.12.self_attn.qkv_proj", + "vision_model.model.layers.12.self_attn.o_proj", + "vision_model.model.layers.12.mlp.fc1", + "vision_model.model.layers.12.mlp.fc2", + "vision_model.model.layers.13.self_attn.qkv_proj", + "vision_model.model.layers.13.self_attn.o_proj", + "vision_model.model.layers.13.mlp.fc1", + "vision_model.model.layers.13.mlp.fc2", + "vision_model.model.layers.14.self_attn.qkv_proj", + "vision_model.model.layers.14.self_attn.o_proj", + "vision_model.model.layers.14.mlp.fc1", + "vision_model.model.layers.14.mlp.fc2", + "vision_model.model.layers.15.self_attn.qkv_proj", + "vision_model.model.layers.15.self_attn.o_proj", + "vision_model.model.layers.15.mlp.fc1", + "vision_model.model.layers.15.mlp.fc2", + "vision_model.model.layers.16.self_attn.qkv_proj", + "vision_model.model.layers.16.self_attn.o_proj", + "vision_model.model.layers.16.mlp.fc1", + "vision_model.model.layers.16.mlp.fc2", + "vision_model.model.layers.17.self_attn.qkv_proj", + "vision_model.model.layers.17.self_attn.o_proj", + "vision_model.model.layers.17.mlp.fc1", + "vision_model.model.layers.17.mlp.fc2", + "vision_model.model.layers.18.self_attn.qkv_proj", + "vision_model.model.layers.18.self_attn.o_proj", + "vision_model.model.layers.18.mlp.fc1", + "vision_model.model.layers.18.mlp.fc2", + "vision_model.model.layers.19.self_attn.qkv_proj", + "vision_model.model.layers.19.self_attn.o_proj", + "vision_model.model.layers.19.mlp.fc1", + "vision_model.model.layers.19.mlp.fc2", + "vision_model.model.layers.20.self_attn.qkv_proj", + "vision_model.model.layers.20.self_attn.o_proj", + "vision_model.model.layers.20.mlp.fc1", + "vision_model.model.layers.20.mlp.fc2", + "vision_model.model.layers.21.self_attn.qkv_proj", + "vision_model.model.layers.21.self_attn.o_proj", + "vision_model.model.layers.21.mlp.fc1", + "vision_model.model.layers.21.mlp.fc2", + "vision_model.model.layers.22.self_attn.qkv_proj", + "vision_model.model.layers.22.self_attn.o_proj", + "vision_model.model.layers.22.mlp.fc1", + "vision_model.model.layers.22.mlp.fc2", + "vision_model.model.layers.23.self_attn.qkv_proj", + "vision_model.model.layers.23.self_attn.o_proj", + "vision_model.model.layers.23.mlp.fc1", + "vision_model.model.layers.23.mlp.fc2", + "vision_model.model.layers.24.self_attn.qkv_proj", + "vision_model.model.layers.24.self_attn.o_proj", + "vision_model.model.layers.24.mlp.fc1", + "vision_model.model.layers.24.mlp.fc2", + "vision_model.model.layers.25.self_attn.qkv_proj", + "vision_model.model.layers.25.self_attn.o_proj", + "vision_model.model.layers.25.mlp.fc1", + "vision_model.model.layers.25.mlp.fc2", + "vision_model.model.layers.26.self_attn.qkv_proj", + "vision_model.model.layers.26.self_attn.o_proj", + "vision_model.model.layers.26.mlp.fc1", + "vision_model.model.layers.26.mlp.fc2", + "vision_model.model.layers.27.self_attn.qkv_proj", + "vision_model.model.layers.27.self_attn.o_proj", + "vision_model.model.layers.27.mlp.fc1", + "vision_model.model.layers.27.mlp.fc2", + "vision_model.model.layers.28.self_attn.qkv_proj", + "vision_model.model.layers.28.self_attn.o_proj", + "vision_model.model.layers.28.mlp.fc1", + "vision_model.model.layers.28.mlp.fc2", + "vision_model.model.layers.29.self_attn.qkv_proj", + "vision_model.model.layers.29.self_attn.o_proj", + "vision_model.model.layers.29.mlp.fc1", + "vision_model.model.layers.29.mlp.fc2", + "vision_model.model.layers.30.self_attn.qkv_proj", + "vision_model.model.layers.30.self_attn.o_proj", + "vision_model.model.layers.30.mlp.fc1", + "vision_model.model.layers.30.mlp.fc2", + "vision_model.model.layers.31.self_attn.qkv_proj", + "vision_model.model.layers.31.self_attn.o_proj", + "vision_model.model.layers.31.mlp.fc1", + "vision_model.model.layers.31.mlp.fc2", + "vision_model.model.layers.32.self_attn.qkv_proj", + "vision_model.model.layers.32.self_attn.o_proj", + "vision_model.model.layers.32.mlp.fc1", + "vision_model.model.layers.32.mlp.fc2", + "vision_model.model.layers.33.self_attn.qkv_proj", + "vision_model.model.layers.33.self_attn.o_proj", + "vision_model.model.layers.33.mlp.fc1", + "vision_model.model.layers.33.mlp.fc2", + "vision_model.vision_adapter.mlp.fc1", + "vision_model.vision_adapter.mlp.fc2", + "multi_modal_projector.linear_1", + "language_model.model.layers.0.self_attn.qkv_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.0.self_attn.attn.impl.softmax", + "language_model.model.layers.0.self_attn.attn.impl.matmul_av", + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.0.self_attn.attn.impl.k_cache", + "language_model.model.layers.0.self_attn.attn.impl.v_cache", + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.0.feed_forward.gate_up_proj", + "language_model.model.layers.0.feed_forward.down_proj", + "language_model.model.layers.1.self_attn.qkv_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.1.self_attn.attn.impl.softmax", + "language_model.model.layers.1.self_attn.attn.impl.matmul_av", + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.1.self_attn.attn.impl.k_cache", + "language_model.model.layers.1.self_attn.attn.impl.v_cache", + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.1.feed_forward.router", + "language_model.model.layers.1.feed_forward.experts", + "language_model.model.layers.1.feed_forward.experts.moe_op", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.1.feed_forward.shared_expert.down_proj", + "language_model.model.layers.2.self_attn.qkv_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.2.self_attn.attn.impl.softmax", + "language_model.model.layers.2.self_attn.attn.impl.matmul_av", + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.2.self_attn.attn.impl.k_cache", + "language_model.model.layers.2.self_attn.attn.impl.v_cache", + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.2.feed_forward.gate_up_proj", + "language_model.model.layers.2.feed_forward.down_proj", + "language_model.model.layers.3.self_attn.qkv_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.3.self_attn.attn.impl.softmax", + "language_model.model.layers.3.self_attn.attn.impl.matmul_av", + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.3.self_attn.attn.impl.k_cache", + "language_model.model.layers.3.self_attn.attn.impl.v_cache", + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.3.feed_forward.router", + "language_model.model.layers.3.feed_forward.experts", + "language_model.model.layers.3.feed_forward.experts.moe_op", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.3.feed_forward.shared_expert.down_proj", + "language_model.model.layers.4.self_attn.qkv_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.4.self_attn.attn.impl.softmax", + "language_model.model.layers.4.self_attn.attn.impl.matmul_av", + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.4.self_attn.attn.impl.k_cache", + "language_model.model.layers.4.self_attn.attn.impl.v_cache", + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.4.feed_forward.gate_up_proj", + "language_model.model.layers.4.feed_forward.down_proj", + "language_model.model.layers.5.self_attn.qkv_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.5.self_attn.attn.impl.softmax", + "language_model.model.layers.5.self_attn.attn.impl.matmul_av", + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.5.self_attn.attn.impl.k_cache", + "language_model.model.layers.5.self_attn.attn.impl.v_cache", + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.5.feed_forward.router", + "language_model.model.layers.5.feed_forward.experts", + "language_model.model.layers.5.feed_forward.experts.moe_op", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.5.feed_forward.shared_expert.down_proj", + "language_model.model.layers.6.self_attn.qkv_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.6.self_attn.attn.impl.softmax", + "language_model.model.layers.6.self_attn.attn.impl.matmul_av", + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.6.self_attn.attn.impl.k_cache", + "language_model.model.layers.6.self_attn.attn.impl.v_cache", + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.6.feed_forward.gate_up_proj", + "language_model.model.layers.6.feed_forward.down_proj", + "language_model.model.layers.7.self_attn.qkv_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.7.self_attn.attn.impl.softmax", + "language_model.model.layers.7.self_attn.attn.impl.matmul_av", + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.7.self_attn.attn.impl.k_cache", + "language_model.model.layers.7.self_attn.attn.impl.v_cache", + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.7.feed_forward.router", + "language_model.model.layers.7.feed_forward.experts", + "language_model.model.layers.7.feed_forward.experts.moe_op", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.7.feed_forward.shared_expert.down_proj", + "language_model.model.layers.8.self_attn.qkv_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.8.self_attn.attn.impl.softmax", + "language_model.model.layers.8.self_attn.attn.impl.matmul_av", + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.8.self_attn.attn.impl.k_cache", + "language_model.model.layers.8.self_attn.attn.impl.v_cache", + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.8.feed_forward.gate_up_proj", + "language_model.model.layers.8.feed_forward.down_proj", + "language_model.model.layers.9.self_attn.qkv_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.9.self_attn.attn.impl.softmax", + "language_model.model.layers.9.self_attn.attn.impl.matmul_av", + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.9.self_attn.attn.impl.k_cache", + "language_model.model.layers.9.self_attn.attn.impl.v_cache", + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.9.feed_forward.router", + "language_model.model.layers.9.feed_forward.experts", + "language_model.model.layers.9.feed_forward.experts.moe_op", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.9.feed_forward.shared_expert.down_proj", + "language_model.model.layers.10.self_attn.qkv_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.10.self_attn.attn.impl.softmax", + "language_model.model.layers.10.self_attn.attn.impl.matmul_av", + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.10.self_attn.attn.impl.k_cache", + "language_model.model.layers.10.self_attn.attn.impl.v_cache", + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.10.feed_forward.gate_up_proj", + "language_model.model.layers.10.feed_forward.down_proj", + "language_model.model.layers.11.self_attn.qkv_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.11.self_attn.attn.impl.softmax", + "language_model.model.layers.11.self_attn.attn.impl.matmul_av", + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.11.self_attn.attn.impl.k_cache", + "language_model.model.layers.11.self_attn.attn.impl.v_cache", + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.11.feed_forward.router", + "language_model.model.layers.11.feed_forward.experts", + "language_model.model.layers.11.feed_forward.experts.moe_op", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.11.feed_forward.shared_expert.down_proj", + "language_model.model.layers.12.self_attn.qkv_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.12.self_attn.attn.impl.softmax", + "language_model.model.layers.12.self_attn.attn.impl.matmul_av", + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.12.self_attn.attn.impl.k_cache", + "language_model.model.layers.12.self_attn.attn.impl.v_cache", + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.12.feed_forward.gate_up_proj", + "language_model.model.layers.12.feed_forward.down_proj", + "language_model.model.layers.13.self_attn.qkv_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.13.self_attn.attn.impl.softmax", + "language_model.model.layers.13.self_attn.attn.impl.matmul_av", + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.13.self_attn.attn.impl.k_cache", + "language_model.model.layers.13.self_attn.attn.impl.v_cache", + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.13.feed_forward.router", + "language_model.model.layers.13.feed_forward.experts", + "language_model.model.layers.13.feed_forward.experts.moe_op", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.13.feed_forward.shared_expert.down_proj", + "language_model.model.layers.14.self_attn.qkv_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.14.self_attn.attn.impl.softmax", + "language_model.model.layers.14.self_attn.attn.impl.matmul_av", + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.14.self_attn.attn.impl.k_cache", + "language_model.model.layers.14.self_attn.attn.impl.v_cache", + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.14.feed_forward.gate_up_proj", + "language_model.model.layers.14.feed_forward.down_proj", + "language_model.model.layers.15.self_attn.qkv_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.15.self_attn.attn.impl.softmax", + "language_model.model.layers.15.self_attn.attn.impl.matmul_av", + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.15.self_attn.attn.impl.k_cache", + "language_model.model.layers.15.self_attn.attn.impl.v_cache", + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.15.feed_forward.router", + "language_model.model.layers.15.feed_forward.experts", + "language_model.model.layers.15.feed_forward.experts.moe_op", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.15.feed_forward.shared_expert.down_proj", + "language_model.model.layers.16.self_attn.qkv_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.16.self_attn.attn.impl.softmax", + "language_model.model.layers.16.self_attn.attn.impl.matmul_av", + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.16.self_attn.attn.impl.k_cache", + "language_model.model.layers.16.self_attn.attn.impl.v_cache", + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.16.feed_forward.gate_up_proj", + "language_model.model.layers.16.feed_forward.down_proj", + "language_model.model.layers.17.self_attn.qkv_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.17.self_attn.attn.impl.softmax", + "language_model.model.layers.17.self_attn.attn.impl.matmul_av", + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.17.self_attn.attn.impl.k_cache", + "language_model.model.layers.17.self_attn.attn.impl.v_cache", + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.17.feed_forward.router", + "language_model.model.layers.17.feed_forward.experts", + "language_model.model.layers.17.feed_forward.experts.moe_op", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.17.feed_forward.shared_expert.down_proj", + "language_model.model.layers.18.self_attn.qkv_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.18.self_attn.attn.impl.softmax", + "language_model.model.layers.18.self_attn.attn.impl.matmul_av", + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.18.self_attn.attn.impl.k_cache", + "language_model.model.layers.18.self_attn.attn.impl.v_cache", + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.18.feed_forward.gate_up_proj", + "language_model.model.layers.18.feed_forward.down_proj", + "language_model.model.layers.19.self_attn.qkv_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.19.self_attn.attn.impl.softmax", + "language_model.model.layers.19.self_attn.attn.impl.matmul_av", + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.19.self_attn.attn.impl.k_cache", + "language_model.model.layers.19.self_attn.attn.impl.v_cache", + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.19.feed_forward.router", + "language_model.model.layers.19.feed_forward.experts", + "language_model.model.layers.19.feed_forward.experts.moe_op", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.19.feed_forward.shared_expert.down_proj", + "language_model.model.layers.20.self_attn.qkv_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.20.self_attn.attn.impl.softmax", + "language_model.model.layers.20.self_attn.attn.impl.matmul_av", + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.20.self_attn.attn.impl.k_cache", + "language_model.model.layers.20.self_attn.attn.impl.v_cache", + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.20.feed_forward.gate_up_proj", + "language_model.model.layers.20.feed_forward.down_proj", + "language_model.model.layers.21.self_attn.qkv_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.21.self_attn.attn.impl.softmax", + "language_model.model.layers.21.self_attn.attn.impl.matmul_av", + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.21.self_attn.attn.impl.k_cache", + "language_model.model.layers.21.self_attn.attn.impl.v_cache", + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.21.feed_forward.router", + "language_model.model.layers.21.feed_forward.experts", + "language_model.model.layers.21.feed_forward.experts.moe_op", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.21.feed_forward.shared_expert.down_proj", + "language_model.model.layers.22.self_attn.qkv_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.22.self_attn.attn.impl.softmax", + "language_model.model.layers.22.self_attn.attn.impl.matmul_av", + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.22.self_attn.attn.impl.k_cache", + "language_model.model.layers.22.self_attn.attn.impl.v_cache", + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.22.feed_forward.gate_up_proj", + "language_model.model.layers.22.feed_forward.down_proj", + "language_model.model.layers.23.self_attn.qkv_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.23.self_attn.attn.impl.softmax", + "language_model.model.layers.23.self_attn.attn.impl.matmul_av", + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.23.self_attn.attn.impl.k_cache", + "language_model.model.layers.23.self_attn.attn.impl.v_cache", + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.23.feed_forward.router", + "language_model.model.layers.23.feed_forward.experts", + "language_model.model.layers.23.feed_forward.experts.moe_op", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.23.feed_forward.shared_expert.down_proj", + "language_model.model.layers.24.self_attn.qkv_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.24.self_attn.attn.impl.softmax", + "language_model.model.layers.24.self_attn.attn.impl.matmul_av", + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.24.self_attn.attn.impl.k_cache", + "language_model.model.layers.24.self_attn.attn.impl.v_cache", + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.24.feed_forward.gate_up_proj", + "language_model.model.layers.24.feed_forward.down_proj", + "language_model.model.layers.25.self_attn.qkv_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.25.self_attn.attn.impl.softmax", + "language_model.model.layers.25.self_attn.attn.impl.matmul_av", + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.25.self_attn.attn.impl.k_cache", + "language_model.model.layers.25.self_attn.attn.impl.v_cache", + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.25.feed_forward.router", + "language_model.model.layers.25.feed_forward.experts", + "language_model.model.layers.25.feed_forward.experts.moe_op", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.25.feed_forward.shared_expert.down_proj", + "language_model.model.layers.26.self_attn.qkv_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.26.self_attn.attn.impl.softmax", + "language_model.model.layers.26.self_attn.attn.impl.matmul_av", + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.26.self_attn.attn.impl.k_cache", + "language_model.model.layers.26.self_attn.attn.impl.v_cache", + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.26.feed_forward.gate_up_proj", + "language_model.model.layers.26.feed_forward.down_proj", + "language_model.model.layers.27.self_attn.qkv_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.27.self_attn.attn.impl.softmax", + "language_model.model.layers.27.self_attn.attn.impl.matmul_av", + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.27.self_attn.attn.impl.k_cache", + "language_model.model.layers.27.self_attn.attn.impl.v_cache", + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.27.feed_forward.router", + "language_model.model.layers.27.feed_forward.experts", + "language_model.model.layers.27.feed_forward.experts.moe_op", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.27.feed_forward.shared_expert.down_proj", + "language_model.model.layers.28.self_attn.qkv_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.28.self_attn.attn.impl.softmax", + "language_model.model.layers.28.self_attn.attn.impl.matmul_av", + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.28.self_attn.attn.impl.k_cache", + "language_model.model.layers.28.self_attn.attn.impl.v_cache", + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.28.feed_forward.gate_up_proj", + "language_model.model.layers.28.feed_forward.down_proj", + "language_model.model.layers.29.self_attn.qkv_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.29.self_attn.attn.impl.softmax", + "language_model.model.layers.29.self_attn.attn.impl.matmul_av", + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.29.self_attn.attn.impl.k_cache", + "language_model.model.layers.29.self_attn.attn.impl.v_cache", + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.29.feed_forward.router", + "language_model.model.layers.29.feed_forward.experts", + "language_model.model.layers.29.feed_forward.experts.moe_op", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.29.feed_forward.shared_expert.down_proj", + "language_model.model.layers.30.self_attn.qkv_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.30.self_attn.attn.impl.softmax", + "language_model.model.layers.30.self_attn.attn.impl.matmul_av", + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.30.self_attn.attn.impl.k_cache", + "language_model.model.layers.30.self_attn.attn.impl.v_cache", + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.30.feed_forward.gate_up_proj", + "language_model.model.layers.30.feed_forward.down_proj", + "language_model.model.layers.31.self_attn.qkv_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.31.self_attn.attn.impl.softmax", + "language_model.model.layers.31.self_attn.attn.impl.matmul_av", + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.31.self_attn.attn.impl.k_cache", + "language_model.model.layers.31.self_attn.attn.impl.v_cache", + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.31.feed_forward.router", + "language_model.model.layers.31.feed_forward.experts", + "language_model.model.layers.31.feed_forward.experts.moe_op", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.31.feed_forward.shared_expert.down_proj", + "language_model.model.layers.32.self_attn.qkv_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.32.self_attn.attn.impl.softmax", + "language_model.model.layers.32.self_attn.attn.impl.matmul_av", + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.32.self_attn.attn.impl.k_cache", + "language_model.model.layers.32.self_attn.attn.impl.v_cache", + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.32.feed_forward.gate_up_proj", + "language_model.model.layers.32.feed_forward.down_proj", + "language_model.model.layers.33.self_attn.qkv_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.33.self_attn.attn.impl.softmax", + "language_model.model.layers.33.self_attn.attn.impl.matmul_av", + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.33.self_attn.attn.impl.k_cache", + "language_model.model.layers.33.self_attn.attn.impl.v_cache", + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.33.feed_forward.router", + "language_model.model.layers.33.feed_forward.experts", + "language_model.model.layers.33.feed_forward.experts.moe_op", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.33.feed_forward.shared_expert.down_proj", + "language_model.model.layers.34.self_attn.qkv_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.34.self_attn.attn.impl.softmax", + "language_model.model.layers.34.self_attn.attn.impl.matmul_av", + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.34.self_attn.attn.impl.k_cache", + "language_model.model.layers.34.self_attn.attn.impl.v_cache", + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.34.feed_forward.gate_up_proj", + "language_model.model.layers.34.feed_forward.down_proj", + "language_model.model.layers.35.self_attn.qkv_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.35.self_attn.attn.impl.softmax", + "language_model.model.layers.35.self_attn.attn.impl.matmul_av", + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.35.self_attn.attn.impl.k_cache", + "language_model.model.layers.35.self_attn.attn.impl.v_cache", + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.35.feed_forward.router", + "language_model.model.layers.35.feed_forward.experts", + "language_model.model.layers.35.feed_forward.experts.moe_op", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.35.feed_forward.shared_expert.down_proj", + "language_model.model.layers.36.self_attn.qkv_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.36.self_attn.attn.impl.softmax", + "language_model.model.layers.36.self_attn.attn.impl.matmul_av", + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.36.self_attn.attn.impl.k_cache", + "language_model.model.layers.36.self_attn.attn.impl.v_cache", + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.36.feed_forward.gate_up_proj", + "language_model.model.layers.36.feed_forward.down_proj", + "language_model.model.layers.37.self_attn.qkv_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.37.self_attn.attn.impl.softmax", + "language_model.model.layers.37.self_attn.attn.impl.matmul_av", + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.37.self_attn.attn.impl.k_cache", + "language_model.model.layers.37.self_attn.attn.impl.v_cache", + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.37.feed_forward.router", + "language_model.model.layers.37.feed_forward.experts", + "language_model.model.layers.37.feed_forward.experts.moe_op", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.37.feed_forward.shared_expert.down_proj", + "language_model.model.layers.38.self_attn.qkv_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.38.self_attn.attn.impl.softmax", + "language_model.model.layers.38.self_attn.attn.impl.matmul_av", + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.38.self_attn.attn.impl.k_cache", + "language_model.model.layers.38.self_attn.attn.impl.v_cache", + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.38.feed_forward.gate_up_proj", + "language_model.model.layers.38.feed_forward.down_proj", + "language_model.model.layers.39.self_attn.qkv_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.39.self_attn.attn.impl.softmax", + "language_model.model.layers.39.self_attn.attn.impl.matmul_av", + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.39.self_attn.attn.impl.k_cache", + "language_model.model.layers.39.self_attn.attn.impl.v_cache", + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.39.feed_forward.router", + "language_model.model.layers.39.feed_forward.experts", + "language_model.model.layers.39.feed_forward.experts.moe_op", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.39.feed_forward.shared_expert.down_proj", + "language_model.model.layers.40.self_attn.qkv_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.40.self_attn.attn.impl.softmax", + "language_model.model.layers.40.self_attn.attn.impl.matmul_av", + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.40.self_attn.attn.impl.k_cache", + "language_model.model.layers.40.self_attn.attn.impl.v_cache", + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.40.feed_forward.gate_up_proj", + "language_model.model.layers.40.feed_forward.down_proj", + "language_model.model.layers.41.self_attn.qkv_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.41.self_attn.attn.impl.softmax", + "language_model.model.layers.41.self_attn.attn.impl.matmul_av", + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.41.self_attn.attn.impl.k_cache", + "language_model.model.layers.41.self_attn.attn.impl.v_cache", + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.41.feed_forward.router", + "language_model.model.layers.41.feed_forward.experts", + "language_model.model.layers.41.feed_forward.experts.moe_op", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.41.feed_forward.shared_expert.down_proj", + "language_model.model.layers.42.self_attn.qkv_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.42.self_attn.attn.impl.softmax", + "language_model.model.layers.42.self_attn.attn.impl.matmul_av", + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.42.self_attn.attn.impl.k_cache", + "language_model.model.layers.42.self_attn.attn.impl.v_cache", + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.42.feed_forward.gate_up_proj", + "language_model.model.layers.42.feed_forward.down_proj", + "language_model.model.layers.43.self_attn.qkv_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.43.self_attn.attn.impl.softmax", + "language_model.model.layers.43.self_attn.attn.impl.matmul_av", + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.43.self_attn.attn.impl.k_cache", + "language_model.model.layers.43.self_attn.attn.impl.v_cache", + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.43.feed_forward.router", + "language_model.model.layers.43.feed_forward.experts", + "language_model.model.layers.43.feed_forward.experts.moe_op", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.43.feed_forward.shared_expert.down_proj", + "language_model.model.layers.44.self_attn.qkv_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.44.self_attn.attn.impl.softmax", + "language_model.model.layers.44.self_attn.attn.impl.matmul_av", + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.44.self_attn.attn.impl.k_cache", + "language_model.model.layers.44.self_attn.attn.impl.v_cache", + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.44.feed_forward.gate_up_proj", + "language_model.model.layers.44.feed_forward.down_proj", + "language_model.model.layers.45.self_attn.qkv_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.45.self_attn.attn.impl.softmax", + "language_model.model.layers.45.self_attn.attn.impl.matmul_av", + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.45.self_attn.attn.impl.k_cache", + "language_model.model.layers.45.self_attn.attn.impl.v_cache", + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.45.feed_forward.router", + "language_model.model.layers.45.feed_forward.experts", + "language_model.model.layers.45.feed_forward.experts.moe_op", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.45.feed_forward.shared_expert.down_proj", + "language_model.model.layers.46.self_attn.qkv_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.46.self_attn.attn.impl.softmax", + "language_model.model.layers.46.self_attn.attn.impl.matmul_av", + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.46.self_attn.attn.impl.k_cache", + "language_model.model.layers.46.self_attn.attn.impl.v_cache", + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.46.feed_forward.gate_up_proj", + "language_model.model.layers.46.feed_forward.down_proj", + "language_model.model.layers.47.self_attn.qkv_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.47.self_attn.attn.impl.softmax", + "language_model.model.layers.47.self_attn.attn.impl.matmul_av", + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.47.self_attn.attn.impl.k_cache", + "language_model.model.layers.47.self_attn.attn.impl.v_cache", + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.47.feed_forward.router", + "language_model.model.layers.47.feed_forward.experts", + "language_model.model.layers.47.feed_forward.experts.moe_op", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.47.feed_forward.shared_expert.down_proj", + "language_model.lm_head" +] \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_1_8.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_1_8.json new file mode 100644 index 000000000000..e34185402ca0 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_1_8.json @@ -0,0 +1 @@ +{"GlobalRank": null, "LocalRank": 1, "Mode": "DynamicRange", "Nodes": {"language_model.model.layers.0.self_attn.qkv_proj": {"inputs": [[[11.1875]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.0.self_attn.o_proj": {"inputs": [[[0.294921875]]], "outputs": [[[0.80078125]], [[3.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.0.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.337890625]], [[4.125]]]}, "language_model.model.layers.0.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.5078125]]]}, "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.337890625]]]}, "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.3359375]]]}, "language_model.model.layers.0.self_attn.attn.impl.k_cache": {"inputs": [[[4.125]]]}, "language_model.model.layers.0.self_attn.attn.impl.v_cache": {"inputs": [[[0.5078125]]]}, "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[4.0625]], [[4.125]], [[0.5078125]]], "outputs": [[[0.294921875]], [[1.0]]]}, "language_model.model.layers.0.feed_forward.gate_up_proj": {"inputs": [[[2.546875]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.0.feed_forward.down_proj": {"inputs": [[[3.34375]]], "outputs": [[[26.75]], [[65.0]]], "params": {"weight": [[0.75]]}}, "language_model.model.layers.1.self_attn.qkv_proj": {"inputs": [[[12.3125]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.1.self_attn.o_proj": {"inputs": [[[0.734375]]], "outputs": [[[0.52734375]], [[1.0078125]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.1.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.76171875]], [[10.8125]]]}, "language_model.model.layers.1.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.8359375]]]}, "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.76171875]]]}, "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.69921875]]]}, "language_model.model.layers.1.self_attn.attn.impl.k_cache": {"inputs": [[[10.8125]]]}, "language_model.model.layers.1.self_attn.attn.impl.v_cache": {"inputs": [[[0.8359375]]]}, "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.0625]], [[9.875]], [[0.83203125]]], "outputs": [[[0.734375]], [[1.0]]]}, "language_model.model.layers.1.feed_forward.router": {"inputs": [[[1.8828125]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op": {"inputs": [[[1.8828125]]], "outputs": [[[1.15625]], [[9.1875]], [[0.0]], [[2.265625]], [[11.375]], [[11.0625]], [[11.9375]], [[16.0]], [[7.46875]], [[3.703125]], [[7.46875]], [[11.3125]], [[12.125]], [[3.421875]], [[11.5]], [[10.25]], [[14.0625]], [[9.6875]], [[8.3125]], [[15.875]], [[14.6875]], [[13.375]], [[10.0]], [[0.0]], [[16.75]], [[7.4375]], [[1.453125]], [[14.0]], [[14.75]], [[12.375]], [[10.0625]], [[21.375]], [[12.625]], [[0.3125]], [[2.375]], [[0.0]], [[12.25]], [[7.34375]], [[11.4375]], [[2.34375]], [[20.875]], [[10.875]], [[10.9375]], [[12.1875]], [[10.125]], [[1.9453125]], [[12.375]], [[12.9375]], [[9.0]], [[3.09375]], [[0.0]], [[9.9375]], [[11.0]], [[10.0625]], [[3.546875]], [[8.9375]], [[18.5]], [[9.625]], [[0.83984375]], [[10.1875]], [[11.25]], [[10.5]], [[10.5625]], [[13.1875]], [[0.0]], [[10.25]], [[6.15625]], [[2.859375]], [[12.5]], [[5.5625]], [[3.890625]], [[10.75]], [[10.4375]], [[12.0]], [[17.375]], [[12.6875]], [[2.515625]], [[10.625]], [[13.625]], [[4.5]], [[21.0]], [[0.0]], [[0.80078125]], [[15.4375]], [[2.828125]], [[16.75]], [[8.5625]], [[13.1875]], [[4.90625]], [[10.625]], [[11.8125]], [[10.5625]], [[20.875]], [[15.1875]], [[0.0]], [[8.5]], [[11.4375]], [[0.11083984375]], [[0.0]], [[12.4375]], [[11.0]], [[11.875]], [[4.21875]], [[13.25]], [[5.5625]], [[0.5546875]], [[5.3125]], [[0.056640625]], [[0.58984375]], [[2.25]], [[0.5390625]], [[10.9375]], [[0.68359375]], [[16.5]], [[0.0]], [[12.9375]], [[13.6875]], [[14.125]], [[8.375]], [[15.625]], [[4.6875]], [[11.625]], [[9.9375]], [[2.984375]], [[4.59375]], [[9.8125]], [[8.5]], [[14.6875]], [[14.8125]]]}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.07470703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.4140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.37109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.357421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[1.8828125]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.1.feed_forward.shared_expert.down_proj": {"inputs": [[[4.3125]]], "outputs": [[[0.88671875]], [[9.238134570705165e+37]]], "params": {"weight": [[0.404296875]]}}, "language_model.model.layers.2.self_attn.qkv_proj": {"inputs": [[[9.875]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.2.self_attn.o_proj": {"inputs": [[[1.0078125]]], "outputs": [[[0.6875]], [[1.328125]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.2.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.57421875]], [[9.5]]]}, "language_model.model.layers.2.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.2421875]]]}, "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.57421875]]]}, "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.796875]]]}, "language_model.model.layers.2.self_attn.attn.impl.k_cache": {"inputs": [[[9.5]]]}, "language_model.model.layers.2.self_attn.attn.impl.v_cache": {"inputs": [[[1.2421875]]]}, "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.0625]], [[9.5]], [[1.2421875]]], "outputs": [[[1.0078125]], [[1.0]]]}, "language_model.model.layers.2.feed_forward.gate_up_proj": {"inputs": [[[6.125]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.2.feed_forward.down_proj": {"inputs": [[[4.28125]]], "outputs": [[[4.75]], [[25.25]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.3.self_attn.qkv_proj": {"inputs": [[[7.21875]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.3.self_attn.o_proj": {"inputs": [[[0.36328125]]], "outputs": [[[0.59765625]], [[2.40625]]], "params": {"weight": [[0.41796875]]}}, "language_model.model.layers.3.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.11181640625]], [[3.578125]]]}, "language_model.model.layers.3.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.39453125]]]}, "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.11181640625]]]}, "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.359375]]]}, "language_model.model.layers.3.self_attn.attn.impl.k_cache": {"inputs": [[[3.578125]]]}, "language_model.model.layers.3.self_attn.attn.impl.v_cache": {"inputs": [[[0.39453125]]]}, "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[1.625]], [[3.578125]], [[0.39453125]]], "outputs": [[[0.36328125]], [[1.0]]]}, "language_model.model.layers.3.feed_forward.router": {"inputs": [[[8.75]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op": {"inputs": [[[8.75]]], "outputs": [[[124.0]], [[0.5625]], [[7.59375]], [[15.0]], [[10.25]], [[0.0]], [[2.234375]], [[19.625]], [[16.5]], [[14.3125]], [[10.9375]], [[3.890625]], [[17.75]], [[11.6875]], [[13.125]], [[8.875]], [[12.5]], [[0.7265625]], [[3.890625]], [[7.375]], [[13.0625]], [[8.125]], [[14.125]], [[15.625]], [[16.0]], [[8.1875]], [[2.171875]], [[11.9375]], [[11.0]], [[7.40625]], [[9.0]], [[11.375]], [[6.46875]], [[13.1875]], [[3.609375]], [[10.75]], [[11.625]], [[2.5]], [[0.79296875]], [[11.5625]], [[4.3125]], [[4.40625]], [[15.5625]], [[1.640625]], [[3.1875]], [[22.25]], [[4.875]], [[17.25]], [[16.75]], [[3.109375]], [[15.75]], [[15.125]], [[10.1875]], [[13.25]], [[11.625]], [[9.25]], [[9.0625]], [[1.6953125]], [[15.0625]], [[12.0]], [[6.28125]], [[3.484375]], [[16.625]], [[14.5]], [[0.4296875]], [[4.96875]], [[11.4375]], [[9.1875]], [[0.16015625]], [[5.1875]], [[0.0]], [[14.125]], [[10.875]], [[10.625]], [[5.28125]], [[5.09375]], [[18.625]], [[11.9375]], [[0.09716796875]], [[11.1875]], [[9.6875]], [[12.0]], [[0.0]], [[3.40625]], [[18.5]], [[8.125]], [[12.6875]], [[8.125]], [[2.453125]], [[18.75]], [[19.375]], [[15.5]], [[6.1875]], [[3.515625]], [[14.6875]], [[0.0]], [[14.625]], [[9.5625]], [[13.625]], [[16.625]], [[4.15625]], [[14.5625]], [[1.09375]], [[4.6875]], [[11.5]], [[0.0]], [[1.7421875]], [[13.625]], [[15.25]], [[11.4375]], [[9.25]], [[17.25]], [[4.53125]], [[15.4375]], [[0.80859375]], [[3.421875]], [[3.234375]], [[8.8125]], [[0.484375]], [[1.265625]], [[2.15625]], [[19.375]], [[9.0]], [[19.375]], [[11.4375]], [[2.453125]], [[14.0]], [[7.4375]], [[10.25]]]}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0751953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0751953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.76171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[8.75]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.3.feed_forward.shared_expert.down_proj": {"inputs": [[[5.09375]]], "outputs": [[[1.015625]], [[9.238134570705165e+37]]], "params": {"weight": [[0.5078125]]}}, "language_model.model.layers.4.self_attn.qkv_proj": {"inputs": [[[13.75]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.4.self_attn.o_proj": {"inputs": [[[1.9140625]]], "outputs": [[[1.015625]], [[2.046875]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.4.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.5234375]], [[10.0]]]}, "language_model.model.layers.4.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.0]]]}, "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.5234375]]]}, "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.453125]]]}, "language_model.model.layers.4.self_attn.attn.impl.k_cache": {"inputs": [[[10.0]]]}, "language_model.model.layers.4.self_attn.attn.impl.v_cache": {"inputs": [[[3.0]]]}, "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[6.125]], [[10.0]], [[3.0]]], "outputs": [[[1.9140625]], [[1.0]]]}, "language_model.model.layers.4.feed_forward.gate_up_proj": {"inputs": [[[7.09375]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.4.feed_forward.down_proj": {"inputs": [[[7.71875]]], "outputs": [[[2.796875]], [[4.96875]]], "params": {"weight": [[0.66015625]]}}, "language_model.model.layers.5.self_attn.qkv_proj": {"inputs": [[[17.375]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.5.self_attn.o_proj": {"inputs": [[[2.34375]]], "outputs": [[[0.97265625]], [[2.21875]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.5.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.59765625]], [[13.375]]]}, "language_model.model.layers.5.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.5625]]]}, "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.59765625]]]}, "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.625]]]}, "language_model.model.layers.5.self_attn.attn.impl.k_cache": {"inputs": [[[13.375]]]}, "language_model.model.layers.5.self_attn.attn.impl.v_cache": {"inputs": [[[3.5625]]]}, "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[7.4375]], [[13.375]], [[3.4375]]], "outputs": [[[2.34375]], [[1.0]]]}, "language_model.model.layers.5.feed_forward.router": {"inputs": [[[19.25]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op": {"inputs": [[[19.25]]], "outputs": [[[170.0]], [[9.5625]], [[2.109375]], [[11.125]], [[20.125]], [[12.875]], [[7.5]], [[18.875]], [[20.5]], [[13.5]], [[17.75]], [[6.59375]], [[15.625]], [[16.0]], [[4.875]], [[21.0]], [[35.5]], [[20.375]], [[11.3125]], [[21.25]], [[6.03125]], [[17.625]], [[20.0]], [[10.375]], [[26.25]], [[2.421875]], [[2.453125]], [[4.5625]], [[189.0]], [[14.0]], [[23.25]], [[18.375]], [[6.1875]], [[13.5625]], [[22.5]], [[19.375]], [[24.875]], [[16.375]], [[5.09375]], [[14.875]], [[13.5625]], [[14.625]], [[5.84375]], [[3.625]], [[16.875]], [[13.9375]], [[13.25]], [[11.9375]], [[15.375]], [[6.3125]], [[15.9375]], [[5.09375]], [[9.0]], [[1.890625]], [[7.53125]], [[16.25]], [[7.21875]], [[7.6875]], [[19.75]], [[22.5]], [[1.7109375]], [[4.5]], [[18.375]], [[6.15625]], [[25.75]], [[6.9375]], [[20.0]], [[15.8125]], [[8.25]], [[14.875]], [[7.1875]], [[4.75]], [[13.125]], [[24.25]], [[18.125]], [[10.75]], [[14.375]], [[17.375]], [[10.9375]], [[18.125]], [[17.25]], [[1.1640625]], [[18.625]], [[6.125]], [[14.5625]], [[5.28125]], [[13.0]], [[0.1826171875]], [[21.75]], [[12.4375]], [[22.5]], [[19.125]], [[9.5]], [[10.75]], [[15.625]], [[20.375]], [[16.875]], [[17.0]], [[21.0]], [[10.0]], [[15.3125]], [[6.6875]], [[2.75]], [[2.546875]], [[34.75]], [[20.625]], [[9.8125]], [[8.625]], [[17.625]], [[10.9375]], [[12.4375]], [[10.125]], [[5.75]], [[3.234375]], [[5.1875]], [[27.375]], [[18.875]], [[2.3125]], [[5.5625]], [[2.671875]], [[17.25]], [[11.8125]], [[2.484375]], [[10.9375]], [[23.875]], [[13.75]], [[7.1875]], [[6.71875]], [[5.21875]]]}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.404296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.78125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.07470703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.69921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.365234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[19.25]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.5.feed_forward.shared_expert.down_proj": {"inputs": [[[6.4375]]], "outputs": [[[0.447265625]], [[9.238134570705165e+37]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.6.self_attn.qkv_proj": {"inputs": [[[12.9375]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.6.self_attn.o_proj": {"inputs": [[[1.484375]]], "outputs": [[[0.66796875]], [[1.5234375]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.6.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8515625]], [[16.625]]]}, "language_model.model.layers.6.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.328125]]]}, "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8515625]]]}, "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.71875]]]}, "language_model.model.layers.6.self_attn.attn.impl.k_cache": {"inputs": [[[16.625]]]}, "language_model.model.layers.6.self_attn.attn.impl.v_cache": {"inputs": [[[2.328125]]]}, "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.8125]], [[16.625]], [[2.328125]]], "outputs": [[[1.484375]], [[1.0]]]}, "language_model.model.layers.6.feed_forward.gate_up_proj": {"inputs": [[[7.71875]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.6.feed_forward.down_proj": {"inputs": [[[7.125]]], "outputs": [[[2.328125]], [[11.1875]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.7.self_attn.qkv_proj": {"inputs": [[[7.09375]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.7.self_attn.o_proj": {"inputs": [[[0.68359375]]], "outputs": [[[1.84375]], [[3.84375]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.7.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.349609375]], [[4.375]]]}, "language_model.model.layers.7.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.140625]]]}, "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.349609375]]]}, "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.82421875]]]}, "language_model.model.layers.7.self_attn.attn.impl.k_cache": {"inputs": [[[4.375]]]}, "language_model.model.layers.7.self_attn.attn.impl.v_cache": {"inputs": [[[1.140625]]]}, "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[4.40625]], [[4.0]], [[1.140625]]], "outputs": [[[0.68359375]], [[1.0]]]}, "language_model.model.layers.7.feed_forward.router": {"inputs": [[[11.6875]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op": {"inputs": [[[11.6875]]], "outputs": [[[2.703125]], [[25.875]], [[26.375]], [[16.75]], [[15.6875]], [[9.375]], [[14.6875]], [[19.875]], [[14.1875]], [[16.5]], [[14.125]], [[16.875]], [[9.25]], [[8.1875]], [[41.75]], [[9.9375]], [[4.03125]], [[16.25]], [[5.625]], [[29.0]], [[17.875]], [[14.25]], [[21.25]], [[18.75]], [[29.0]], [[13.375]], [[18.75]], [[9.125]], [[4.875]], [[11.375]], [[9.0]], [[12.6875]], [[22.375]], [[23.0]], [[6.90625]], [[7.21875]], [[7.46875]], [[27.875]], [[13.25]], [[31.625]], [[3.109375]], [[0.87890625]], [[6.59375]], [[2.3125]], [[21.75]], [[19.25]], [[25.75]], [[0.87109375]], [[19.5]], [[21.125]], [[29.75]], [[16.5]], [[13.125]], [[18.0]], [[5.25]], [[26.625]], [[33.75]], [[7.71875]], [[2.125]], [[6.53125]], [[5.8125]], [[13.125]], [[20.125]], [[21.625]], [[14.4375]], [[32.25]], [[20.0]], [[8.875]], [[11.5]], [[6.28125]], [[2.21875]], [[27.0]], [[3.453125]], [[14.4375]], [[8.75]], [[30.0]], [[7.09375]], [[4.78125]], [[5.5]], [[1.15625]], [[9.9375]], [[17.875]], [[14.25]], [[5.03125]], [[12.6875]], [[6.625]], [[15.0]], [[3.640625]], [[26.625]], [[5.75]], [[40.75]], [[25.625]], [[12.625]], [[10.5]], [[4.375]], [[26.75]], [[3.828125]], [[22.25]], [[5.6875]], [[23.25]], [[3.4375]], [[12.9375]], [[17.0]], [[9.375]], [[1.0078125]], [[22.875]], [[30.125]], [[17.625]], [[10.875]], [[0.9140625]], [[28.5]], [[21.625]], [[24.875]], [[10.25]], [[32.75]], [[15.3125]], [[29.25]], [[6.0625]], [[12.0]], [[4.75]], [[15.6875]], [[23.375]], [[29.875]], [[19.375]], [[7.78125]], [[3.53125]], [[2.125]], [[21.5]], [[11.875]]]}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.07470703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.6171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.55859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[11.6875]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.7.feed_forward.shared_expert.down_proj": {"inputs": [[[8.0625]]], "outputs": [[[0.6796875]], [[9.238134570705165e+37]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.8.self_attn.qkv_proj": {"inputs": [[[15.125]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.8.self_attn.o_proj": {"inputs": [[[1.03125]]], "outputs": [[[1.9375]], [[2.109375]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.8.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8125]], [[12.625]]]}, "language_model.model.layers.8.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.9375]]]}, "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8125]]]}, "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.2890625]]]}, "language_model.model.layers.8.self_attn.attn.impl.k_cache": {"inputs": [[[12.625]]]}, "language_model.model.layers.8.self_attn.attn.impl.v_cache": {"inputs": [[[2.9375]]]}, "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.875]], [[12.625]], [[2.9375]]], "outputs": [[[1.03125]], [[1.0]]]}, "language_model.model.layers.8.feed_forward.gate_up_proj": {"inputs": [[[8.25]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.8.feed_forward.down_proj": {"inputs": [[[8.8125]]], "outputs": [[[2.5625]], [[9.25]]], "params": {"weight": [[0.50390625]]}}, "language_model.model.layers.9.self_attn.qkv_proj": {"inputs": [[[15.75]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.9.self_attn.o_proj": {"inputs": [[[2.171875]]], "outputs": [[[1.3125]], [[2.875]]], "params": {"weight": [[0.388671875]]}}, "language_model.model.layers.9.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8671875]], [[15.75]]]}, "language_model.model.layers.9.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.90625]]]}, "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8671875]]]}, "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.140625]]]}, "language_model.model.layers.9.self_attn.attn.impl.k_cache": {"inputs": [[[15.75]]]}, "language_model.model.layers.9.self_attn.attn.impl.v_cache": {"inputs": [[[3.90625]]]}, "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.125]], [[15.75]], [[3.90625]]], "outputs": [[[2.078125]], [[1.0]]]}, "language_model.model.layers.9.feed_forward.router": {"inputs": [[[11.75]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op": {"inputs": [[[11.75]]], "outputs": [[[3.140625]], [[16.125]], [[20.625]], [[2.890625]], [[39.0]], [[34.5]], [[35.75]], [[29.375]], [[21.125]], [[10.75]], [[44.75]], [[38.5]], [[15.8125]], [[42.75]], [[24.75]], [[12.5625]], [[7.125]], [[17.125]], [[19.0]], [[36.25]], [[9.6875]], [[14.5]], [[15.375]], [[9.5]], [[18.875]], [[11.0625]], [[21.75]], [[39.75]], [[39.75]], [[13.25]], [[47.5]], [[14.4375]], [[18.75]], [[27.875]], [[21.5]], [[38.0]], [[7.09375]], [[29.75]], [[17.25]], [[7.8125]], [[16.0]], [[6.78125]], [[8.625]], [[21.125]], [[36.25]], [[12.4375]], [[13.75]], [[14.75]], [[18.125]], [[19.375]], [[31.625]], [[14.0]], [[24.75]], [[13.75]], [[12.75]], [[26.125]], [[5.25]], [[30.0]], [[23.625]], [[29.875]], [[21.75]], [[25.75]], [[9.625]], [[22.5]], [[28.625]], [[16.25]], [[14.3125]], [[30.5]], [[51.25]], [[3.875]], [[14.4375]], [[34.0]], [[44.75]], [[8.4375]], [[18.875]], [[3.65625]], [[26.0]], [[11.25]], [[45.25]], [[5.3125]], [[13.1875]], [[42.25]], [[23.25]], [[29.375]], [[16.75]], [[27.375]], [[39.5]], [[33.5]], [[30.875]], [[27.125]], [[45.5]], [[9.1875]], [[7.21875]], [[19.75]], [[8.125]], [[30.75]], [[53.25]], [[15.875]], [[14.3125]], [[10.4375]], [[12.75]], [[32.25]], [[13.5]], [[39.0]], [[11.0625]], [[17.75]], [[36.75]], [[23.75]], [[23.625]], [[14.3125]], [[25.75]], [[22.625]], [[2.609375]], [[25.0]], [[7.9375]], [[38.25]], [[43.75]], [[10.125]], [[3.5]], [[25.125]], [[7.84375]], [[40.25]], [[6.09375]], [[26.625]], [[22.625]], [[56.0]], [[18.375]], [[10.75]], [[12.125]]]}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0751953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[11.75]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.9.feed_forward.shared_expert.down_proj": {"inputs": [[[11.375]]], "outputs": [[[2.90625]], [[9.238134570705165e+37]]], "params": {"weight": [[0.6015625]]}}, "language_model.model.layers.10.self_attn.qkv_proj": {"inputs": [[[16.125]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.10.self_attn.o_proj": {"inputs": [[[3.71875]]], "outputs": [[[1.6015625]], [[3.09375]]], "params": {"weight": [[0.60546875]]}}, "language_model.model.layers.10.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.984375]], [[9.6875]]]}, "language_model.model.layers.10.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.96875]]]}, "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.984375]]]}, "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.34375]]]}, "language_model.model.layers.10.self_attn.attn.impl.k_cache": {"inputs": [[[9.6875]]]}, "language_model.model.layers.10.self_attn.attn.impl.v_cache": {"inputs": [[[4.96875]]]}, "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[17.75]], [[9.6875]], [[4.96875]]], "outputs": [[[3.71875]], [[1.0]]]}, "language_model.model.layers.10.feed_forward.gate_up_proj": {"inputs": [[[9.875]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.10.feed_forward.down_proj": {"inputs": [[[12.9375]]], "outputs": [[[6.90625]], [[11.3125]]], "params": {"weight": [[0.34765625]]}}, "language_model.model.layers.11.self_attn.qkv_proj": {"inputs": [[[13.5625]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.11.self_attn.o_proj": {"inputs": [[[3.359375]]], "outputs": [[[8.0625]], [[7.875]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.11.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.5546875]], [[8.6875]]]}, "language_model.model.layers.11.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.03125]]]}, "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.5546875]]]}, "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.71875]]]}, "language_model.model.layers.11.self_attn.attn.impl.k_cache": {"inputs": [[[8.6875]]]}, "language_model.model.layers.11.self_attn.attn.impl.v_cache": {"inputs": [[[5.03125]]]}, "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[7.1875]], [[8.6875]], [[5.03125]]], "outputs": [[[3.359375]], [[1.0]]]}, "language_model.model.layers.11.feed_forward.router": {"inputs": [[[13.375]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op": {"inputs": [[[13.375]]], "outputs": [[[3.84375]], [[29.0]], [[17.375]], [[23.875]], [[10.375]], [[19.75]], [[30.125]], [[1.2578125]], [[12.0625]], [[16.875]], [[5.75]], [[27.75]], [[15.6875]], [[53.75]], [[39.25]], [[22.5]], [[38.0]], [[43.25]], [[34.75]], [[57.75]], [[30.875]], [[11.9375]], [[33.25]], [[29.5]], [[18.875]], [[25.375]], [[43.75]], [[28.5]], [[23.0]], [[15.375]], [[42.25]], [[31.75]], [[8.5]], [[6.75]], [[9.4375]], [[31.25]], [[26.125]], [[19.25]], [[22.375]], [[13.375]], [[14.3125]], [[3.78125]], [[14.1875]], [[5.6875]], [[29.375]], [[65.5]], [[3.21875]], [[18.25]], [[30.375]], [[24.5]], [[7.90625]], [[33.0]], [[21.75]], [[4.28125]], [[45.5]], [[54.75]], [[41.75]], [[36.0]], [[16.75]], [[29.625]], [[20.0]], [[18.75]], [[17.125]], [[26.25]], [[24.25]], [[18.5]], [[25.5]], [[21.375]], [[48.0]], [[13.875]], [[38.25]], [[4.375]], [[7.6875]], [[2.03125]], [[39.0]], [[15.875]], [[36.25]], [[17.0]], [[5.0625]], [[32.75]], [[45.25]], [[13.9375]], [[11.875]], [[19.0]], [[23.5]], [[2.828125]], [[4.3125]], [[31.875]], [[6.625]], [[8.0625]], [[30.25]], [[36.25]], [[6.9375]], [[27.0]], [[40.0]], [[9.75]], [[20.25]], [[26.875]], [[33.5]], [[11.375]], [[27.875]], [[6.03125]], [[42.75]], [[23.0]], [[20.625]], [[28.5]], [[32.5]], [[17.5]], [[6.40625]], [[13.5625]], [[14.6875]], [[14.1875]], [[4.25]], [[5.65625]], [[11.375]], [[34.0]], [[36.5]], [[42.5]], [[38.0]], [[25.125]], [[31.875]], [[30.75]], [[16.125]], [[28.375]], [[12.9375]], [[7.5]], [[10.6875]], [[34.5]], [[19.25]]]}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.41796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[13.375]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.11.feed_forward.shared_expert.down_proj": {"inputs": [[[11.9375]]], "outputs": [[[3.421875]], [[9.238134570705165e+37]]], "params": {"weight": [[0.65234375]]}}, "language_model.model.layers.12.self_attn.qkv_proj": {"inputs": [[[18.625]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.12.self_attn.o_proj": {"inputs": [[[2.75]]], "outputs": [[[1.3359375]], [[3.640625]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.12.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.9296875]], [[16.75]]]}, "language_model.model.layers.12.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.390625]]]}, "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.9296875]]]}, "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.1875]]]}, "language_model.model.layers.12.self_attn.attn.impl.k_cache": {"inputs": [[[16.75]]]}, "language_model.model.layers.12.self_attn.attn.impl.v_cache": {"inputs": [[[3.390625]]]}, "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.125]], [[16.75]], [[3.390625]]], "outputs": [[[2.75]], [[1.0]]]}, "language_model.model.layers.12.feed_forward.gate_up_proj": {"inputs": [[[10.75]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.12.feed_forward.down_proj": {"inputs": [[[15.5625]]], "outputs": [[[9.0]], [[17.375]]], "params": {"weight": [[0.8671875]]}}, "language_model.model.layers.13.self_attn.qkv_proj": {"inputs": [[[20.875]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.13.self_attn.o_proj": {"inputs": [[[2.78125]]], "outputs": [[[1.6875]], [[6.9375]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.13.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.9375]], [[16.375]]]}, "language_model.model.layers.13.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.953125]]]}, "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.9375]]]}, "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.046875]]]}, "language_model.model.layers.13.self_attn.attn.impl.k_cache": {"inputs": [[[16.375]]]}, "language_model.model.layers.13.self_attn.attn.impl.v_cache": {"inputs": [[[2.953125]]]}, "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.5]], [[16.375]], [[2.953125]]], "outputs": [[[2.78125]], [[1.0]]]}, "language_model.model.layers.13.feed_forward.router": {"inputs": [[[17.625]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op": {"inputs": [[[17.625]]], "outputs": [[[4.65625]], [[22.75]], [[60.75]], [[14.9375]], [[52.0]], [[50.0]], [[13.875]], [[8.5625]], [[17.0]], [[28.25]], [[2.203125]], [[28.375]], [[29.5]], [[16.125]], [[21.25]], [[20.375]], [[24.875]], [[11.625]], [[30.25]], [[10.875]], [[19.25]], [[21.125]], [[43.5]], [[10.0625]], [[3.328125]], [[20.5]], [[42.0]], [[11.5625]], [[44.75]], [[48.0]], [[13.625]], [[15.375]], [[28.125]], [[28.875]], [[38.75]], [[50.25]], [[20.625]], [[37.25]], [[45.0]], [[16.0]], [[5.90625]], [[27.875]], [[34.25]], [[20.375]], [[29.375]], [[42.75]], [[55.75]], [[24.375]], [[18.625]], [[50.0]], [[47.0]], [[28.25]], [[42.5]], [[12.8125]], [[64.5]], [[6.0]], [[31.875]], [[35.25]], [[71.5]], [[25.125]], [[55.5]], [[6.5]], [[31.375]], [[6.34375]], [[31.0]], [[9.25]], [[45.25]], [[11.5625]], [[23.25]], [[16.125]], [[10.6875]], [[50.75]], [[77.0]], [[25.875]], [[11.6875]], [[24.375]], [[33.0]], [[31.0]], [[26.625]], [[25.5]], [[14.9375]], [[10.1875]], [[38.75]], [[6.03125]], [[48.0]], [[34.0]], [[43.75]], [[38.5]], [[9.25]], [[4.96875]], [[47.5]], [[54.5]], [[46.25]], [[2.265625]], [[60.25]], [[43.75]], [[15.75]], [[10.8125]], [[14.0625]], [[17.75]], [[52.25]], [[5.75]], [[2.34375]], [[27.375]], [[25.625]], [[51.0]], [[33.0]], [[62.0]], [[16.5]], [[13.4375]], [[35.0]], [[36.5]], [[28.5]], [[12.5625]], [[21.125]], [[21.375]], [[32.0]], [[18.0]], [[41.5]], [[49.0]], [[33.25]], [[26.125]], [[34.5]], [[28.375]], [[15.3125]], [[19.0]], [[18.625]], [[5.9375]], [[19.875]]]}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.50390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[17.625]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.13.feed_forward.shared_expert.down_proj": {"inputs": [[[21.125]]], "outputs": [[[1.2890625]], [[9.238134570705165e+37]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.14.self_attn.qkv_proj": {"inputs": [[[18.75]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.14.self_attn.o_proj": {"inputs": [[[3.6875]]], "outputs": [[[2.140625]], [[4.875]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.14.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.73046875]], [[8.375]]]}, "language_model.model.layers.14.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.625]]]}, "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.73046875]]]}, "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.140625]]]}, "language_model.model.layers.14.self_attn.attn.impl.k_cache": {"inputs": [[[8.375]]]}, "language_model.model.layers.14.self_attn.attn.impl.v_cache": {"inputs": [[[4.625]]]}, "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.1875]], [[8.375]], [[4.625]]], "outputs": [[[3.6875]], [[1.0]]]}, "language_model.model.layers.14.feed_forward.gate_up_proj": {"inputs": [[[12.0625]]], "params": {"weight": [[0.43359375]]}}, "language_model.model.layers.14.feed_forward.down_proj": {"inputs": [[[19.375]]], "outputs": [[[4.65625]], [[28.25]]], "params": {"weight": [[0.42578125]]}}, "language_model.model.layers.15.self_attn.qkv_proj": {"inputs": [[[20.875]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.15.self_attn.o_proj": {"inputs": [[[4.6875]]], "outputs": [[[21.875]], [[23.0]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.15.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.828125]], [[11.0]]]}, "language_model.model.layers.15.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.28125]]]}, "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.828125]]]}, "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.234375]]]}, "language_model.model.layers.15.self_attn.attn.impl.k_cache": {"inputs": [[[11.0]]]}, "language_model.model.layers.15.self_attn.attn.impl.v_cache": {"inputs": [[[5.28125]]]}, "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.8125]], [[11.0]], [[5.28125]]], "outputs": [[[4.6875]], [[1.0]]]}, "language_model.model.layers.15.feed_forward.router": {"inputs": [[[19.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op": {"inputs": [[[19.0]]], "outputs": [[[4.96875]], [[8.5]], [[14.4375]], [[17.375]], [[25.625]], [[29.875]], [[73.5]], [[32.5]], [[38.5]], [[60.25]], [[60.75]], [[20.625]], [[25.625]], [[21.875]], [[19.5]], [[4.71875]], [[21.625]], [[19.625]], [[24.625]], [[29.0]], [[8.5625]], [[31.75]], [[71.5]], [[43.0]], [[49.75]], [[15.8125]], [[6.65625]], [[21.625]], [[23.125]], [[35.25]], [[33.0]], [[38.25]], [[23.125]], [[10.625]], [[10.5]], [[32.0]], [[21.875]], [[10.6875]], [[33.25]], [[41.5]], [[73.5]], [[43.25]], [[41.75]], [[67.0]], [[46.5]], [[45.0]], [[22.25]], [[8.4375]], [[56.75]], [[19.375]], [[16.75]], [[36.25]], [[41.5]], [[33.0]], [[32.75]], [[53.0]], [[26.375]], [[25.25]], [[35.75]], [[49.75]], [[38.75]], [[16.5]], [[70.5]], [[90.0]], [[40.25]], [[26.25]], [[48.25]], [[53.0]], [[44.25]], [[11.5625]], [[50.5]], [[29.5]], [[13.75]], [[26.375]], [[66.5]], [[47.75]], [[25.125]], [[44.5]], [[61.25]], [[59.0]], [[4.9375]], [[24.375]], [[28.25]], [[17.25]], [[9.25]], [[21.25]], [[26.125]], [[41.75]], [[11.875]], [[37.5]], [[36.0]], [[5.4375]], [[38.75]], [[18.375]], [[65.5]], [[44.75]], [[29.0]], [[36.75]], [[30.0]], [[38.0]], [[58.5]], [[25.25]], [[44.5]], [[30.25]], [[36.5]], [[34.25]], [[31.875]], [[9.25]], [[26.125]], [[21.875]], [[12.5]], [[47.25]], [[43.75]], [[63.5]], [[8.875]], [[20.0]], [[55.0]], [[27.875]], [[9.5625]], [[50.75]], [[36.0]], [[76.0]], [[53.5]], [[17.5]], [[30.0]], [[37.5]], [[12.25]], [[15.3125]], [[31.5]]]}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[19.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.15.feed_forward.shared_expert.down_proj": {"inputs": [[[20.75]]], "outputs": [[[9.5]], [[9.238134570705165e+37]]], "params": {"weight": [[0.4453125]]}}, "language_model.model.layers.16.self_attn.qkv_proj": {"inputs": [[[20.375]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.16.self_attn.o_proj": {"inputs": [[[3.75]]], "outputs": [[[2.75]], [[6.28125]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.16.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0546875]], [[17.25]]]}, "language_model.model.layers.16.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.09375]]]}, "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0546875]]]}, "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.875]]]}, "language_model.model.layers.16.self_attn.attn.impl.k_cache": {"inputs": [[[17.25]]]}, "language_model.model.layers.16.self_attn.attn.impl.v_cache": {"inputs": [[[4.09375]]]}, "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.5]], [[17.25]], [[4.09375]]], "outputs": [[[3.75]], [[1.0]]]}, "language_model.model.layers.16.feed_forward.gate_up_proj": {"inputs": [[[14.8125]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.16.feed_forward.down_proj": {"inputs": [[[17.0]]], "outputs": [[[7.03125]], [[22.625]]], "params": {"weight": [[0.6796875]]}}, "language_model.model.layers.17.self_attn.qkv_proj": {"inputs": [[[19.875]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.17.self_attn.o_proj": {"inputs": [[[4.84375]]], "outputs": [[[17.5]], [[20.75]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.17.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.9453125]], [[12.5625]]]}, "language_model.model.layers.17.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.21875]]]}, "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.9453125]]]}, "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.03125]]]}, "language_model.model.layers.17.self_attn.attn.impl.k_cache": {"inputs": [[[12.5625]]]}, "language_model.model.layers.17.self_attn.attn.impl.v_cache": {"inputs": [[[5.21875]]]}, "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.4375]], [[12.5625]], [[5.21875]]], "outputs": [[[4.84375]], [[1.0]]]}, "language_model.model.layers.17.feed_forward.router": {"inputs": [[[23.875]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op": {"inputs": [[[23.875]]], "outputs": [[[5.34375]], [[26.375]], [[26.125]], [[29.0]], [[15.875]], [[61.0]], [[20.875]], [[53.0]], [[32.5]], [[44.25]], [[5.90625]], [[9.125]], [[15.625]], [[20.875]], [[24.25]], [[42.5]], [[37.5]], [[32.75]], [[41.25]], [[18.25]], [[27.75]], [[58.0]], [[27.125]], [[38.75]], [[40.5]], [[22.625]], [[52.0]], [[16.25]], [[15.3125]], [[35.5]], [[26.375]], [[46.5]], [[42.0]], [[16.375]], [[42.5]], [[27.875]], [[56.5]], [[21.75]], [[21.875]], [[16.0]], [[38.75]], [[25.375]], [[76.5]], [[10.0625]], [[21.0]], [[35.5]], [[58.5]], [[35.25]], [[71.5]], [[48.25]], [[28.125]], [[54.0]], [[39.25]], [[12.125]], [[16.625]], [[15.25]], [[5.8125]], [[14.5625]], [[38.25]], [[49.0]], [[16.625]], [[7.1875]], [[32.0]], [[8.75]], [[18.5]], [[47.5]], [[52.25]], [[14.625]], [[51.0]], [[37.0]], [[13.75]], [[32.25]], [[18.75]], [[64.0]], [[40.25]], [[19.125]], [[45.0]], [[53.0]], [[53.75]], [[13.0625]], [[39.25]], [[14.8125]], [[52.0]], [[45.5]], [[48.0]], [[73.5]], [[43.25]], [[17.5]], [[27.25]], [[34.25]], [[51.5]], [[36.0]], [[50.0]], [[6.4375]], [[50.5]], [[11.25]], [[65.0]], [[7.0625]], [[4.28125]], [[24.25]], [[28.5]], [[33.0]], [[19.0]], [[42.75]], [[41.25]], [[35.5]], [[17.125]], [[15.8125]], [[61.75]], [[17.0]], [[14.6875]], [[48.25]], [[35.0]], [[45.0]], [[20.5]], [[38.25]], [[17.75]], [[42.0]], [[28.0]], [[25.25]], [[43.0]], [[67.5]], [[38.5]], [[53.25]], [[13.4375]], [[14.9375]], [[5.0625]], [[9.8125]], [[78.0]]]}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.875]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.17.feed_forward.shared_expert.down_proj": {"inputs": [[[19.875]]], "outputs": [[[6.53125]], [[9.238134570705165e+37]]], "params": {"weight": [[0.44140625]]}}, "language_model.model.layers.18.self_attn.qkv_proj": {"inputs": [[[22.125]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.18.self_attn.o_proj": {"inputs": [[[3.578125]]], "outputs": [[[6.375]], [[11.5625]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.18.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.9296875]], [[17.25]]]}, "language_model.model.layers.18.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.96875]]]}, "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.9296875]]]}, "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.1875]]]}, "language_model.model.layers.18.self_attn.attn.impl.k_cache": {"inputs": [[[17.25]]]}, "language_model.model.layers.18.self_attn.attn.impl.v_cache": {"inputs": [[[4.96875]]]}, "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.0]], [[17.25]], [[4.96875]]], "outputs": [[[3.578125]], [[1.0]]]}, "language_model.model.layers.18.feed_forward.gate_up_proj": {"inputs": [[[18.5]]], "params": {"weight": [[0.4140625]]}}, "language_model.model.layers.18.feed_forward.down_proj": {"inputs": [[[14.5]]], "outputs": [[[8.25]], [[358.0]]], "params": {"weight": [[0.75]]}}, "language_model.model.layers.19.self_attn.qkv_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.19.self_attn.o_proj": {"inputs": [[[4.71875]]], "outputs": [[[31.625]], [[36.75]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.19.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.81640625]], [[11.8125]]]}, "language_model.model.layers.19.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.09375]]]}, "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.81640625]]]}, "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.34375]]]}, "language_model.model.layers.19.self_attn.attn.impl.k_cache": {"inputs": [[[11.8125]]]}, "language_model.model.layers.19.self_attn.attn.impl.v_cache": {"inputs": [[[6.09375]]]}, "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.8125]], [[11.8125]], [[6.09375]]], "outputs": [[[4.71875]], [[1.0]]]}, "language_model.model.layers.19.feed_forward.router": {"inputs": [[[23.25]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op": {"inputs": [[[23.25]]], "outputs": [[[5.65625]], [[41.0]], [[11.125]], [[15.0625]], [[54.0]], [[56.0]], [[15.375]], [[12.375]], [[46.5]], [[21.875]], [[8.125]], [[8.75]], [[30.25]], [[24.875]], [[34.25]], [[34.0]], [[8.625]], [[10.0]], [[28.0]], [[40.75]], [[16.75]], [[12.4375]], [[22.5]], [[29.875]], [[17.5]], [[27.25]], [[30.875]], [[58.75]], [[60.0]], [[46.5]], [[31.5]], [[59.25]], [[52.0]], [[33.0]], [[10.4375]], [[29.5]], [[35.75]], [[19.125]], [[26.875]], [[14.875]], [[46.5]], [[39.5]], [[14.75]], [[44.5]], [[44.5]], [[37.75]], [[11.8125]], [[13.625]], [[14.5625]], [[27.125]], [[45.5]], [[11.375]], [[42.25]], [[6.875]], [[45.75]], [[36.0]], [[18.75]], [[25.125]], [[37.0]], [[53.75]], [[44.0]], [[33.5]], [[46.5]], [[12.6875]], [[13.625]], [[35.5]], [[74.5]], [[15.3125]], [[21.375]], [[33.0]], [[9.875]], [[29.625]], [[30.625]], [[66.5]], [[24.875]], [[22.5]], [[14.0625]], [[20.875]], [[33.0]], [[45.5]], [[7.84375]], [[36.0]], [[19.875]], [[40.0]], [[45.25]], [[20.375]], [[41.0]], [[22.125]], [[13.6875]], [[23.75]], [[19.0]], [[23.875]], [[42.0]], [[33.5]], [[47.25]], [[46.25]], [[25.125]], [[32.5]], [[31.625]], [[24.875]], [[11.125]], [[93.0]], [[26.625]], [[12.9375]], [[37.0]], [[24.0]], [[22.375]], [[47.75]], [[25.0]], [[36.25]], [[38.0]], [[10.875]], [[11.6875]], [[41.25]], [[39.25]], [[15.0625]], [[8.8125]], [[30.125]], [[53.25]], [[20.0]], [[49.5]], [[22.375]], [[28.0]], [[11.125]], [[27.5]], [[9.625]], [[19.0]], [[38.25]], [[38.0]]]}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.408203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.384765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.39453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.25]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.19.feed_forward.shared_expert.down_proj": {"inputs": [[[17.875]]], "outputs": [[[3.296875]], [[9.238134570705165e+37]]], "params": {"weight": [[0.3984375]]}}, "language_model.model.layers.20.self_attn.qkv_proj": {"inputs": [[[52.5]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.20.self_attn.o_proj": {"inputs": [[[6.625]]], "outputs": [[[9.5]], [[18.375]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.20.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0234375]], [[17.125]]]}, "language_model.model.layers.20.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.15625]]]}, "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0234375]]]}, "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.71875]]]}, "language_model.model.layers.20.self_attn.attn.impl.k_cache": {"inputs": [[[17.125]]]}, "language_model.model.layers.20.self_attn.attn.impl.v_cache": {"inputs": [[[7.15625]]]}, "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.3125]], [[17.125]], [[7.15625]]], "outputs": [[[6.625]], [[1.0]]]}, "language_model.model.layers.20.feed_forward.gate_up_proj": {"inputs": [[[41.5]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.20.feed_forward.down_proj": {"inputs": [[[17.625]]], "outputs": [[[9.4375]], [[38.5]]], "params": {"weight": [[0.91015625]]}}, "language_model.model.layers.21.self_attn.qkv_proj": {"inputs": [[[56.25]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.21.self_attn.o_proj": {"inputs": [[[5.5625]]], "outputs": [[[25.125]], [[37.75]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.21.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0859375]], [[17.875]]]}, "language_model.model.layers.21.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.625]]]}, "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0859375]]]}, "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.125]]]}, "language_model.model.layers.21.self_attn.attn.impl.k_cache": {"inputs": [[[17.875]]]}, "language_model.model.layers.21.self_attn.attn.impl.v_cache": {"inputs": [[[6.625]]]}, "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.8125]], [[17.875]], [[6.625]]], "outputs": [[[5.5625]], [[1.0]]]}, "language_model.model.layers.21.feed_forward.router": {"inputs": [[[24.5]]], "params": {"weight": [[0.45703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op": {"inputs": [[[24.5]]], "outputs": [[[4.46875]], [[16.875]], [[53.5]], [[29.25]], [[60.5]], [[16.75]], [[31.0]], [[20.5]], [[25.875]], [[26.5]], [[28.125]], [[20.125]], [[7.09375]], [[27.5]], [[57.0]], [[15.0]], [[33.75]], [[38.0]], [[31.375]], [[22.375]], [[29.25]], [[38.0]], [[32.25]], [[20.75]], [[14.3125]], [[31.375]], [[18.0]], [[10.625]], [[13.6875]], [[12.6875]], [[35.75]], [[9.1875]], [[54.5]], [[46.75]], [[15.5]], [[12.5625]], [[10.9375]], [[26.5]], [[28.625]], [[36.25]], [[49.75]], [[8.125]], [[27.75]], [[19.125]], [[8.3125]], [[26.875]], [[58.5]], [[6.90625]], [[35.0]], [[38.75]], [[25.625]], [[53.0]], [[39.75]], [[30.25]], [[37.75]], [[11.375]], [[46.5]], [[31.625]], [[13.5]], [[10.125]], [[31.75]], [[32.75]], [[19.875]], [[13.875]], [[96.5]], [[10.3125]], [[58.25]], [[38.0]], [[23.5]], [[9.875]], [[27.375]], [[25.625]], [[21.375]], [[34.25]], [[29.375]], [[27.75]], [[10.8125]], [[37.5]], [[21.375]], [[17.25]], [[8.5]], [[33.5]], [[22.25]], [[48.25]], [[39.25]], [[21.625]], [[54.5]], [[9.125]], [[15.3125]], [[9.0]], [[16.25]], [[20.0]], [[36.25]], [[65.5]], [[10.9375]], [[25.125]], [[19.625]], [[25.5]], [[33.0]], [[25.875]], [[26.5]], [[19.375]], [[27.5]], [[27.5]], [[77.0]], [[26.0]], [[24.875]], [[20.625]], [[37.0]], [[32.0]], [[19.75]], [[34.5]], [[31.875]], [[17.375]], [[12.875]], [[20.625]], [[13.0]], [[51.25]], [[34.75]], [[12.875]], [[59.25]], [[6.5625]], [[26.375]], [[45.25]], [[19.875]], [[17.125]], [[51.75]], [[19.5]], [[47.0]]]}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.365234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[24.5]]], "params": {"weight": [[0.416015625]]}}, "language_model.model.layers.21.feed_forward.shared_expert.down_proj": {"inputs": [[[25.5]]], "outputs": [[[3.59375]], [[9.238134570705165e+37]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.22.self_attn.qkv_proj": {"inputs": [[[55.75]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.22.self_attn.o_proj": {"inputs": [[[3.828125]]], "outputs": [[[17.25]], [[23.25]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.22.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.03125]], [[18.375]]]}, "language_model.model.layers.22.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.8125]]]}, "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.03125]]]}, "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.546875]]]}, "language_model.model.layers.22.self_attn.attn.impl.k_cache": {"inputs": [[[18.375]]]}, "language_model.model.layers.22.self_attn.attn.impl.v_cache": {"inputs": [[[5.65625]]]}, "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.5625]], [[18.375]], [[5.65625]]], "outputs": [[[3.828125]], [[1.0]]]}, "language_model.model.layers.22.feed_forward.gate_up_proj": {"inputs": [[[44.75]]], "params": {"weight": [[0.361328125]]}}, "language_model.model.layers.22.feed_forward.down_proj": {"inputs": [[[17.125]]], "outputs": [[[12.0]], [[80.5]]], "params": {"weight": [[0.78515625]]}}, "language_model.model.layers.23.self_attn.qkv_proj": {"inputs": [[[67.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.23.self_attn.o_proj": {"inputs": [[[6.71875]]], "outputs": [[[50.5]], [[61.5]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.23.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8828125]], [[13.0]]]}, "language_model.model.layers.23.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[13.0]]]}, "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8828125]]]}, "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.5625]]]}, "language_model.model.layers.23.self_attn.attn.impl.k_cache": {"inputs": [[[13.0]]]}, "language_model.model.layers.23.self_attn.attn.impl.v_cache": {"inputs": [[[13.0]]]}, "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.9375]], [[13.0]], [[13.0]]], "outputs": [[[6.71875]], [[1.0]]]}, "language_model.model.layers.23.feed_forward.router": {"inputs": [[[23.625]]], "params": {"weight": [[0.474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op": {"inputs": [[[23.625]]], "outputs": [[[6.09375]], [[55.5]], [[9.0]], [[49.25]], [[45.0]], [[8.375]], [[50.75]], [[22.5]], [[34.25]], [[96.5]], [[14.5]], [[34.75]], [[63.75]], [[43.0]], [[10.8125]], [[18.875]], [[63.25]], [[31.25]], [[23.5]], [[28.375]], [[34.0]], [[20.375]], [[42.5]], [[7.09375]], [[13.625]], [[42.5]], [[10.0625]], [[42.75]], [[96.5]], [[19.75]], [[23.125]], [[17.5]], [[24.0]], [[18.5]], [[26.625]], [[46.25]], [[35.0]], [[58.75]], [[17.625]], [[24.375]], [[33.0]], [[59.5]], [[16.5]], [[76.5]], [[12.625]], [[17.875]], [[8.375]], [[54.0]], [[34.0]], [[39.0]], [[67.5]], [[12.5625]], [[53.5]], [[24.875]], [[21.75]], [[15.8125]], [[21.0]], [[41.0]], [[11.875]], [[62.0]], [[16.25]], [[32.5]], [[31.375]], [[9.6875]], [[9.5625]], [[11.5]], [[11.6875]], [[39.75]], [[35.5]], [[44.5]], [[38.0]], [[6.09375]], [[23.875]], [[64.0]], [[42.25]], [[25.375]], [[32.75]], [[33.25]], [[58.75]], [[49.75]], [[20.5]], [[33.75]], [[25.875]], [[84.5]], [[24.375]], [[10.9375]], [[49.25]], [[14.875]], [[8.375]], [[68.5]], [[17.625]], [[30.75]], [[46.75]], [[23.375]], [[11.875]], [[31.5]], [[71.0]], [[20.625]], [[42.25]], [[16.125]], [[36.5]], [[16.625]], [[12.375]], [[11.875]], [[14.6875]], [[26.375]], [[78.0]], [[29.875]], [[39.0]], [[9.8125]], [[95.5]], [[7.46875]], [[94.5]], [[12.9375]], [[78.0]], [[20.875]], [[19.75]], [[21.5]], [[65.5]], [[13.4375]], [[15.9375]], [[15.875]], [[38.5]], [[28.0]], [[54.75]], [[92.5]], [[21.25]], [[12.3125]], [[9.375]]]}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.625]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.23.feed_forward.shared_expert.down_proj": {"inputs": [[[20.0]]], "outputs": [[[6.65625]], [[9.238134570705165e+37]]], "params": {"weight": [[0.431640625]]}}, "language_model.model.layers.24.self_attn.qkv_proj": {"inputs": [[[60.5]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.24.self_attn.o_proj": {"inputs": [[[5.4375]]], "outputs": [[[36.0]], [[42.5]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.24.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1875]], [[12.5]]]}, "language_model.model.layers.24.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.65625]]]}, "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1875]]]}, "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.4375]]]}, "language_model.model.layers.24.self_attn.attn.impl.k_cache": {"inputs": [[[12.5]]]}, "language_model.model.layers.24.self_attn.attn.impl.v_cache": {"inputs": [[[6.65625]]]}, "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.3125]], [[12.5]], [[6.65625]]], "outputs": [[[4.375]], [[1.0]]]}, "language_model.model.layers.24.feed_forward.gate_up_proj": {"inputs": [[[40.5]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.24.feed_forward.down_proj": {"inputs": [[[101.0]]], "outputs": [[[13.3125]], [[62.5]]], "params": {"weight": [[0.4921875]]}}, "language_model.model.layers.25.self_attn.qkv_proj": {"inputs": [[[49.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.25.self_attn.o_proj": {"inputs": [[[6.40625]]], "outputs": [[[47.75]], [[57.0]]], "params": {"weight": [[0.65234375]]}}, "language_model.model.layers.25.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1015625]], [[17.125]]]}, "language_model.model.layers.25.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.625]]]}, "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1015625]]]}, "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.375]]]}, "language_model.model.layers.25.self_attn.attn.impl.k_cache": {"inputs": [[[17.125]]]}, "language_model.model.layers.25.self_attn.attn.impl.v_cache": {"inputs": [[[9.625]]]}, "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.75]], [[17.125]], [[9.625]]], "outputs": [[[6.40625]], [[1.0]]]}, "language_model.model.layers.25.feed_forward.router": {"inputs": [[[30.375]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op": {"inputs": [[[30.375]]], "outputs": [[[7.21875]], [[57.75]], [[33.75]], [[29.25]], [[11.375]], [[37.0]], [[49.25]], [[21.375]], [[28.75]], [[27.75]], [[18.75]], [[25.875]], [[20.75]], [[31.25]], [[60.0]], [[46.25]], [[26.875]], [[23.25]], [[15.5]], [[60.5]], [[39.25]], [[32.75]], [[46.0]], [[46.5]], [[31.625]], [[13.5]], [[12.0]], [[52.0]], [[15.3125]], [[7.28125]], [[38.75]], [[22.375]], [[12.75]], [[11.1875]], [[61.75]], [[17.5]], [[7.65625]], [[23.875]], [[27.5]], [[16.5]], [[50.5]], [[38.75]], [[52.75]], [[40.0]], [[36.5]], [[16.25]], [[37.0]], [[86.0]], [[10.4375]], [[12.1875]], [[32.75]], [[46.5]], [[12.0625]], [[13.1875]], [[9.4375]], [[37.25]], [[50.25]], [[57.75]], [[15.375]], [[38.25]], [[11.1875]], [[23.625]], [[63.75]], [[20.0]], [[66.0]], [[116.0]], [[13.1875]], [[56.5]], [[32.25]], [[26.75]], [[15.4375]], [[18.25]], [[23.875]], [[36.75]], [[23.125]], [[75.5]], [[72.5]], [[12.0]], [[9.375]], [[33.25]], [[44.25]], [[22.5]], [[46.75]], [[38.5]], [[30.75]], [[10.5625]], [[18.875]], [[33.5]], [[40.25]], [[21.0]], [[40.0]], [[9.6875]], [[26.625]], [[36.5]], [[43.75]], [[8.125]], [[32.0]], [[38.0]], [[41.25]], [[11.125]], [[29.25]], [[44.75]], [[25.125]], [[22.5]], [[14.25]], [[19.125]], [[21.25]], [[51.0]], [[5.375]], [[45.25]], [[34.75]], [[44.5]], [[23.0]], [[39.0]], [[49.0]], [[98.5]], [[40.25]], [[78.5]], [[13.6875]], [[63.0]], [[28.5]], [[34.75]], [[31.5]], [[53.25]], [[15.8125]], [[30.75]], [[7.4375]], [[23.875]], [[10.625]]]}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[30.375]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.25.feed_forward.shared_expert.down_proj": {"inputs": [[[26.25]]], "outputs": [[[5.375]], [[9.238134570705165e+37]]], "params": {"weight": [[0.57421875]]}}, "language_model.model.layers.26.self_attn.qkv_proj": {"inputs": [[[63.25]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.26.self_attn.o_proj": {"inputs": [[[6.1875]]], "outputs": [[[29.0]], [[53.0]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.26.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0703125]], [[16.625]]]}, "language_model.model.layers.26.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.5625]]]}, "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0703125]]]}, "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.1875]]]}, "language_model.model.layers.26.self_attn.attn.impl.k_cache": {"inputs": [[[16.625]]]}, "language_model.model.layers.26.self_attn.attn.impl.v_cache": {"inputs": [[[8.5625]]]}, "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.375]], [[16.625]], [[8.5625]]], "outputs": [[[5.78125]], [[1.0]]]}, "language_model.model.layers.26.feed_forward.gate_up_proj": {"inputs": [[[45.5]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.26.feed_forward.down_proj": {"inputs": [[[26.0]]], "outputs": [[[15.25]], [[103.0]]], "params": {"weight": [[0.734375]]}}, "language_model.model.layers.27.self_attn.qkv_proj": {"inputs": [[[72.5]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.27.self_attn.o_proj": {"inputs": [[[6.3125]]], "outputs": [[[60.0]], [[66.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.27.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1796875]], [[14.25]]]}, "language_model.model.layers.27.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[13.3125]]]}, "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1796875]]]}, "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.09375]]]}, "language_model.model.layers.27.self_attn.attn.impl.k_cache": {"inputs": [[[14.25]]]}, "language_model.model.layers.27.self_attn.attn.impl.v_cache": {"inputs": [[[13.3125]]]}, "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.0]], [[14.25]], [[13.3125]]], "outputs": [[[6.25]], [[1.0]]]}, "language_model.model.layers.27.feed_forward.router": {"inputs": [[[30.875]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op": {"inputs": [[[30.875]]], "outputs": [[[11.875]], [[16.0]], [[47.5]], [[44.5]], [[23.625]], [[15.4375]], [[45.0]], [[60.5]], [[36.0]], [[40.75]], [[46.25]], [[30.875]], [[26.875]], [[56.25]], [[18.25]], [[64.0]], [[22.5]], [[15.5625]], [[63.25]], [[23.875]], [[22.375]], [[34.0]], [[12.375]], [[62.25]], [[26.25]], [[28.25]], [[15.125]], [[68.5]], [[16.0]], [[13.875]], [[28.75]], [[76.5]], [[15.25]], [[50.5]], [[17.375]], [[65.5]], [[19.5]], [[17.0]], [[66.0]], [[42.75]], [[20.0]], [[28.0]], [[35.0]], [[101.5]], [[57.75]], [[16.25]], [[42.75]], [[8.625]], [[27.875]], [[33.25]], [[11.8125]], [[32.25]], [[11.375]], [[77.0]], [[47.0]], [[35.75]], [[52.75]], [[28.5]], [[28.25]], [[52.25]], [[22.0]], [[33.75]], [[49.0]], [[24.5]], [[13.125]], [[44.5]], [[111.0]], [[16.375]], [[16.625]], [[14.4375]], [[14.6875]], [[23.75]], [[25.5]], [[111.0]], [[40.0]], [[38.0]], [[70.0]], [[27.875]], [[40.5]], [[39.75]], [[39.75]], [[57.75]], [[35.0]], [[19.375]], [[50.75]], [[30.75]], [[22.5]], [[13.4375]], [[28.0]], [[15.25]], [[30.25]], [[12.125]], [[54.75]], [[18.625]], [[42.5]], [[31.25]], [[35.25]], [[38.0]], [[92.0]], [[14.5625]], [[34.0]], [[61.75]], [[36.0]], [[14.3125]], [[36.75]], [[22.0]], [[10.625]], [[11.125]], [[33.5]], [[15.25]], [[37.0]], [[15.625]], [[50.0]], [[11.25]], [[18.25]], [[21.5]], [[20.5]], [[40.75]], [[30.125]], [[48.5]], [[176.0]], [[12.4375]], [[33.25]], [[61.5]], [[109.0]], [[13.9375]], [[28.5]], [[30.125]], [[38.75]]]}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.41015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.34765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[30.875]]], "params": {"weight": [[0.5]]}}, "language_model.model.layers.27.feed_forward.shared_expert.down_proj": {"inputs": [[[42.5]]], "outputs": [[[34.75]], [[9.238134570705165e+37]]], "params": {"weight": [[0.87109375]]}}, "language_model.model.layers.28.self_attn.qkv_proj": {"inputs": [[[77.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.28.self_attn.o_proj": {"inputs": [[[4.71875]]], "outputs": [[[72.0]], [[74.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.28.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.390625]], [[16.75]]]}, "language_model.model.layers.28.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.03125]]]}, "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.390625]]]}, "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.71875]]]}, "language_model.model.layers.28.self_attn.attn.impl.k_cache": {"inputs": [[[16.75]]]}, "language_model.model.layers.28.self_attn.attn.impl.v_cache": {"inputs": [[[7.03125]]]}, "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[17.25]], [[16.75]], [[7.03125]]], "outputs": [[[3.796875]], [[1.0]]]}, "language_model.model.layers.28.feed_forward.gate_up_proj": {"inputs": [[[40.25]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.28.feed_forward.down_proj": {"inputs": [[[44.75]]], "outputs": [[[95.0]], [[103.5]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.29.self_attn.qkv_proj": {"inputs": [[[66.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.29.self_attn.o_proj": {"inputs": [[[5.0625]]], "outputs": [[[70.5]], [[76.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.29.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.125]], [[19.125]]]}, "language_model.model.layers.29.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.40625]]]}, "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.125]]]}, "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.90625]]]}, "language_model.model.layers.29.self_attn.attn.impl.k_cache": {"inputs": [[[19.125]]]}, "language_model.model.layers.29.self_attn.attn.impl.v_cache": {"inputs": [[[7.40625]]]}, "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.25]], [[19.125]], [[7.40625]]], "outputs": [[[5.0625]], [[1.0]]]}, "language_model.model.layers.29.feed_forward.router": {"inputs": [[[31.75]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op": {"inputs": [[[31.75]]], "outputs": [[[6.1875]], [[25.75]], [[43.0]], [[54.25]], [[32.75]], [[26.125]], [[45.5]], [[17.75]], [[50.5]], [[34.0]], [[40.75]], [[11.4375]], [[12.75]], [[11.3125]], [[35.5]], [[23.25]], [[11.0]], [[62.5]], [[27.375]], [[16.5]], [[25.75]], [[40.25]], [[29.375]], [[34.0]], [[10.875]], [[76.5]], [[25.0]], [[40.5]], [[25.375]], [[19.125]], [[58.25]], [[30.0]], [[11.25]], [[39.75]], [[68.0]], [[11.9375]], [[24.875]], [[32.25]], [[58.5]], [[53.5]], [[44.5]], [[18.125]], [[21.25]], [[19.0]], [[56.75]], [[21.375]], [[30.75]], [[22.0]], [[41.0]], [[46.0]], [[25.125]], [[34.5]], [[44.75]], [[48.5]], [[62.0]], [[25.5]], [[34.25]], [[27.375]], [[42.25]], [[38.0]], [[15.875]], [[26.875]], [[10.375]], [[21.0]], [[17.0]], [[69.0]], [[52.75]], [[30.125]], [[33.0]], [[45.25]], [[15.5]], [[25.125]], [[44.75]], [[14.5625]], [[26.875]], [[45.5]], [[56.25]], [[20.875]], [[17.75]], [[74.0]], [[21.25]], [[22.5]], [[15.25]], [[32.75]], [[44.25]], [[31.875]], [[27.0]], [[38.0]], [[12.3125]], [[21.625]], [[14.3125]], [[26.5]], [[34.5]], [[28.0]], [[16.5]], [[9.625]], [[41.25]], [[21.875]], [[15.375]], [[68.0]], [[17.125]], [[35.25]], [[31.625]], [[52.75]], [[43.25]], [[12.875]], [[98.0]], [[48.75]], [[24.5]], [[13.875]], [[24.875]], [[21.125]], [[39.0]], [[61.5]], [[19.0]], [[22.75]], [[33.5]], [[31.5]], [[38.75]], [[61.25]], [[27.75]], [[44.75]], [[26.5]], [[56.5]], [[92.0]], [[15.3125]], [[25.625]], [[102.0]], [[53.25]]]}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.7421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.37109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[31.75]]], "params": {"weight": [[0.447265625]]}}, "language_model.model.layers.29.feed_forward.shared_expert.down_proj": {"inputs": [[[48.75]]], "outputs": [[[75.5]], [[9.238134570705165e+37]]], "params": {"weight": [[0.875]]}}, "language_model.model.layers.30.self_attn.qkv_proj": {"inputs": [[[71.5]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.30.self_attn.o_proj": {"inputs": [[[4.6875]]], "outputs": [[[52.25]], [[57.0]]], "params": {"weight": [[0.59375]]}}, "language_model.model.layers.30.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1484375]], [[17.625]]]}, "language_model.model.layers.30.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.65625]]]}, "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1484375]]]}, "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.71875]]]}, "language_model.model.layers.30.self_attn.attn.impl.k_cache": {"inputs": [[[17.625]]]}, "language_model.model.layers.30.self_attn.attn.impl.v_cache": {"inputs": [[[7.65625]]]}, "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.75]], [[17.625]], [[7.65625]]], "outputs": [[[4.6875]], [[1.0]]]}, "language_model.model.layers.30.feed_forward.gate_up_proj": {"inputs": [[[32.0]]], "params": {"weight": [[0.5078125]]}}, "language_model.model.layers.30.feed_forward.down_proj": {"inputs": [[[31.75]]], "outputs": [[[18.625]], [[139.0]]], "params": {"weight": [[0.447265625]]}}, "language_model.model.layers.31.self_attn.qkv_proj": {"inputs": [[[71.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.31.self_attn.o_proj": {"inputs": [[[4.78125]]], "outputs": [[[66.5]], [[88.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.31.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.91015625]], [[10.4375]]]}, "language_model.model.layers.31.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.5625]]]}, "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.91015625]]]}, "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.84375]]]}, "language_model.model.layers.31.self_attn.attn.impl.k_cache": {"inputs": [[[10.4375]]]}, "language_model.model.layers.31.self_attn.attn.impl.v_cache": {"inputs": [[[9.6875]]]}, "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.25]], [[10.4375]], [[9.6875]]], "outputs": [[[4.78125]], [[1.0]]]}, "language_model.model.layers.31.feed_forward.router": {"inputs": [[[33.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op": {"inputs": [[[33.0]]], "outputs": [[[11.5625]], [[15.6875]], [[107.0]], [[22.25]], [[25.375]], [[60.5]], [[42.0]], [[52.0]], [[13.1875]], [[30.0]], [[76.0]], [[31.25]], [[13.6875]], [[37.25]], [[13.5625]], [[35.0]], [[16.75]], [[22.125]], [[20.5]], [[84.0]], [[29.0]], [[8.1875]], [[36.5]], [[33.75]], [[90.5]], [[50.5]], [[77.5]], [[48.0]], [[44.0]], [[38.5]], [[14.8125]], [[20.125]], [[35.5]], [[28.75]], [[71.0]], [[17.625]], [[17.375]], [[33.5]], [[27.0]], [[23.0]], [[66.0]], [[120.0]], [[48.25]], [[23.625]], [[57.75]], [[67.5]], [[61.25]], [[49.75]], [[35.5]], [[85.5]], [[28.875]], [[38.75]], [[41.5]], [[70.0]], [[15.9375]], [[52.0]], [[22.75]], [[20.875]], [[47.75]], [[29.0]], [[46.0]], [[24.125]], [[35.75]], [[22.5]], [[65.5]], [[43.75]], [[41.25]], [[26.125]], [[51.25]], [[39.0]], [[20.5]], [[29.375]], [[48.5]], [[17.25]], [[30.875]], [[13.6875]], [[12.1875]], [[37.5]], [[45.0]], [[31.0]], [[27.375]], [[25.25]], [[119.5]], [[23.5]], [[87.5]], [[29.5]], [[36.75]], [[66.5]], [[28.125]], [[57.0]], [[61.75]], [[60.5]], [[27.25]], [[30.75]], [[12.5625]], [[62.5]], [[28.25]], [[39.25]], [[18.875]], [[25.75]], [[26.0]], [[26.125]], [[8.75]], [[38.0]], [[38.5]], [[57.5]], [[9.625]], [[23.375]], [[14.5625]], [[23.25]], [[60.75]], [[39.0]], [[66.0]], [[36.5]], [[34.75]], [[26.25]], [[40.5]], [[71.5]], [[17.25]], [[26.5]], [[49.25]], [[21.0]], [[26.0]], [[10.625]], [[12.4375]], [[100.0]], [[17.125]], [[17.75]], [[38.25]]]}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.4140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[33.0]]], "params": {"weight": [[0.404296875]]}}, "language_model.model.layers.31.feed_forward.shared_expert.down_proj": {"inputs": [[[43.5]]], "outputs": [[[11.1875]], [[9.238134570705165e+37]]], "params": {"weight": [[0.53515625]]}}, "language_model.model.layers.32.self_attn.qkv_proj": {"inputs": [[[53.25]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.32.self_attn.o_proj": {"inputs": [[[4.84375]]], "outputs": [[[96.0]], [[105.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.32.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.265625]], [[15.5625]]]}, "language_model.model.layers.32.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.65625]]]}, "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.265625]]]}, "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.53125]]]}, "language_model.model.layers.32.self_attn.attn.impl.k_cache": {"inputs": [[[15.5625]]]}, "language_model.model.layers.32.self_attn.attn.impl.v_cache": {"inputs": [[[6.65625]]]}, "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[16.375]], [[15.5625]], [[6.65625]]], "outputs": [[[4.84375]], [[1.0]]]}, "language_model.model.layers.32.feed_forward.gate_up_proj": {"inputs": [[[32.75]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.32.feed_forward.down_proj": {"inputs": [[[46.25]]], "outputs": [[[21.125]], [[79.5]]], "params": {"weight": [[0.453125]]}}, "language_model.model.layers.33.self_attn.qkv_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.33.self_attn.o_proj": {"inputs": [[[10.875]]], "outputs": [[[97.0]], [[118.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.33.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.3671875]], [[14.25]]]}, "language_model.model.layers.33.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.25]]]}, "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.3671875]]]}, "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.71875]]]}, "language_model.model.layers.33.self_attn.attn.impl.k_cache": {"inputs": [[[14.25]]]}, "language_model.model.layers.33.self_attn.attn.impl.v_cache": {"inputs": [[[12.25]]]}, "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.0]], [[14.25]], [[12.25]]], "outputs": [[[10.875]], [[1.0]]]}, "language_model.model.layers.33.feed_forward.router": {"inputs": [[[64.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op": {"inputs": [[[64.0]]], "outputs": [[[9.1875]], [[47.75]], [[24.25]], [[67.0]], [[29.875]], [[55.0]], [[97.0]], [[15.625]], [[124.0]], [[33.5]], [[37.25]], [[29.0]], [[59.75]], [[57.5]], [[24.75]], [[60.0]], [[41.5]], [[26.875]], [[99.0]], [[33.5]], [[33.5]], [[26.25]], [[25.0]], [[22.5]], [[78.0]], [[33.75]], [[21.375]], [[92.5]], [[42.25]], [[52.5]], [[95.0]], [[24.5]], [[121.0]], [[20.25]], [[106.0]], [[36.5]], [[20.75]], [[34.0]], [[27.25]], [[79.5]], [[31.625]], [[28.375]], [[28.625]], [[86.5]], [[69.5]], [[45.0]], [[20.375]], [[100.0]], [[26.25]], [[56.0]], [[73.5]], [[43.75]], [[13.5]], [[21.375]], [[18.5]], [[138.0]], [[91.5]], [[60.5]], [[20.0]], [[40.5]], [[84.5]], [[31.375]], [[39.5]], [[63.5]], [[74.5]], [[22.0]], [[51.0]], [[41.0]], [[18.875]], [[52.25]], [[54.5]], [[48.75]], [[63.5]], [[73.5]], [[66.5]], [[26.375]], [[50.0]], [[41.25]], [[20.375]], [[13.8125]], [[32.5]], [[25.5]], [[11.5]], [[89.5]], [[31.0]], [[20.25]], [[64.5]], [[53.25]], [[18.75]], [[130.0]], [[36.0]], [[12.75]], [[36.25]], [[20.875]], [[61.25]], [[68.5]], [[102.0]], [[42.75]], [[87.0]], [[55.5]], [[69.5]], [[74.5]], [[92.0]], [[36.5]], [[29.75]], [[58.75]], [[40.75]], [[66.5]], [[26.0]], [[31.125]], [[11.5]], [[36.75]], [[118.0]], [[18.5]], [[41.25]], [[72.5]], [[19.375]], [[15.875]], [[34.75]], [[18.25]], [[88.0]], [[27.75]], [[30.5]], [[17.625]], [[8.0625]], [[46.0]], [[27.625]], [[14.375]], [[48.5]]]}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.35546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.41796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.423828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[64.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.33.feed_forward.shared_expert.down_proj": {"inputs": [[[67.0]]], "outputs": [[[13.25]], [[9.238134570705165e+37]]], "params": {"weight": [[0.48828125]]}}, "language_model.model.layers.34.self_attn.qkv_proj": {"inputs": [[[54.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.34.self_attn.o_proj": {"inputs": [[[6.46875]]], "outputs": [[[46.75]], [[80.5]]], "params": {"weight": [[0.99609375]]}}, "language_model.model.layers.34.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.265625]], [[16.125]]]}, "language_model.model.layers.34.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.21875]]]}, "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.265625]]]}, "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.125]]]}, "language_model.model.layers.34.self_attn.attn.impl.k_cache": {"inputs": [[[16.125]]]}, "language_model.model.layers.34.self_attn.attn.impl.v_cache": {"inputs": [[[7.21875]]]}, "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.875]], [[16.125]], [[7.21875]]], "outputs": [[[6.46875]], [[1.0]]]}, "language_model.model.layers.34.feed_forward.gate_up_proj": {"inputs": [[[31.125]]], "params": {"weight": [[0.6640625]]}}, "language_model.model.layers.34.feed_forward.down_proj": {"inputs": [[[59.0]]], "outputs": [[[30.375]], [[128.0]]], "params": {"weight": [[0.671875]]}}, "language_model.model.layers.35.self_attn.qkv_proj": {"inputs": [[[52.75]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.35.self_attn.o_proj": {"inputs": [[[9.3125]]], "outputs": [[[87.5]], [[93.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.35.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.421875]], [[13.3125]]]}, "language_model.model.layers.35.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.3125]]]}, "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.421875]]]}, "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[9.3125]]]}, "language_model.model.layers.35.self_attn.attn.impl.k_cache": {"inputs": [[[13.3125]]]}, "language_model.model.layers.35.self_attn.attn.impl.v_cache": {"inputs": [[[12.3125]]]}, "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[16.75]], [[13.3125]], [[12.3125]]], "outputs": [[[7.78125]], [[1.0]]]}, "language_model.model.layers.35.feed_forward.router": {"inputs": [[[64.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op": {"inputs": [[[64.0]]], "outputs": [[[11.125]], [[121.5]], [[31.25]], [[26.5]], [[30.5]], [[125.5]], [[24.75]], [[76.5]], [[86.0]], [[77.0]], [[48.75]], [[65.5]], [[24.625]], [[88.5]], [[42.25]], [[77.0]], [[59.75]], [[71.0]], [[46.25]], [[54.5]], [[58.75]], [[61.75]], [[36.5]], [[29.625]], [[103.0]], [[38.75]], [[55.25]], [[92.0]], [[45.5]], [[33.25]], [[46.25]], [[76.0]], [[41.75]], [[58.0]], [[36.75]], [[41.75]], [[117.5]], [[44.25]], [[39.5]], [[32.5]], [[39.0]], [[69.5]], [[55.0]], [[44.25]], [[39.0]], [[95.0]], [[44.0]], [[22.25]], [[26.75]], [[59.0]], [[71.0]], [[39.5]], [[37.5]], [[66.0]], [[44.75]], [[27.375]], [[30.125]], [[121.0]], [[34.25]], [[88.0]], [[116.0]], [[72.5]], [[32.5]], [[43.5]], [[30.25]], [[33.5]], [[20.75]], [[112.5]], [[64.5]], [[25.5]], [[107.5]], [[50.25]], [[159.0]], [[32.25]], [[91.0]], [[53.5]], [[56.0]], [[51.0]], [[76.0]], [[92.0]], [[96.0]], [[93.5]], [[62.75]], [[24.5]], [[67.0]], [[71.5]], [[167.0]], [[45.25]], [[28.5]], [[20.0]], [[34.5]], [[116.0]], [[54.5]], [[62.25]], [[116.5]], [[84.0]], [[79.0]], [[71.5]], [[47.0]], [[125.5]], [[30.625]], [[112.0]], [[26.625]], [[52.75]], [[44.75]], [[14.1875]], [[70.0]], [[44.5]], [[27.5]], [[120.5]], [[95.5]], [[63.5]], [[41.5]], [[52.0]], [[34.25]], [[75.5]], [[65.5]], [[57.25]], [[22.625]], [[43.0]], [[66.0]], [[42.0]], [[18.375]], [[33.0]], [[105.0]], [[61.25]], [[27.75]], [[18.5]], [[55.75]]]}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.462890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.396484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.4296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[64.0]]], "params": {"weight": [[0.48046875]]}}, "language_model.model.layers.35.feed_forward.shared_expert.down_proj": {"inputs": [[[93.5]]], "outputs": [[[11.5625]], [[9.238134570705165e+37]]], "params": {"weight": [[0.51171875]]}}, "language_model.model.layers.36.self_attn.qkv_proj": {"inputs": [[[56.75]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.36.self_attn.o_proj": {"inputs": [[[7.5]]], "outputs": [[[27.75]], [[43.0]]], "params": {"weight": [[0.79296875]]}}, "language_model.model.layers.36.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.140625]], [[16.25]]]}, "language_model.model.layers.36.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[10.375]]]}, "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.140625]]]}, "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.78125]]]}, "language_model.model.layers.36.self_attn.attn.impl.k_cache": {"inputs": [[[16.25]]]}, "language_model.model.layers.36.self_attn.attn.impl.v_cache": {"inputs": [[[10.375]]]}, "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.5]], [[16.25]], [[10.375]]], "outputs": [[[7.5]], [[1.0]]]}, "language_model.model.layers.36.feed_forward.gate_up_proj": {"inputs": [[[29.625]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.36.feed_forward.down_proj": {"inputs": [[[68.5]]], "outputs": [[[38.25]], [[76.5]]], "params": {"weight": [[0.59375]]}}, "language_model.model.layers.37.self_attn.qkv_proj": {"inputs": [[[50.25]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.37.self_attn.o_proj": {"inputs": [[[5.84375]]], "outputs": [[[102.0]], [[105.0]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.37.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0078125]], [[23.125]]]}, "language_model.model.layers.37.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.5625]]]}, "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0078125]]]}, "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.0625]]]}, "language_model.model.layers.37.self_attn.attn.impl.k_cache": {"inputs": [[[23.125]]]}, "language_model.model.layers.37.self_attn.attn.impl.v_cache": {"inputs": [[[6.5625]]]}, "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.1875]], [[23.125]], [[6.5625]]], "outputs": [[[5.84375]], [[1.0]]]}, "language_model.model.layers.37.feed_forward.router": {"inputs": [[[70.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op": {"inputs": [[[70.0]]], "outputs": [[[12.125]], [[77.0]], [[79.0]], [[40.5]], [[114.5]], [[60.5]], [[35.0]], [[36.5]], [[79.5]], [[27.125]], [[65.0]], [[42.5]], [[29.5]], [[36.0]], [[77.5]], [[71.5]], [[38.25]], [[58.5]], [[41.0]], [[62.5]], [[49.5]], [[62.25]], [[92.5]], [[112.0]], [[26.875]], [[104.5]], [[97.0]], [[62.5]], [[68.0]], [[139.0]], [[57.25]], [[22.25]], [[115.0]], [[40.75]], [[30.75]], [[72.5]], [[48.0]], [[86.0]], [[102.0]], [[75.0]], [[85.0]], [[59.75]], [[39.25]], [[26.875]], [[45.5]], [[33.25]], [[102.0]], [[134.0]], [[25.5]], [[64.0]], [[18.625]], [[62.5]], [[59.75]], [[92.0]], [[67.0]], [[37.0]], [[76.0]], [[89.0]], [[41.75]], [[72.0]], [[119.5]], [[92.5]], [[69.0]], [[58.75]], [[31.125]], [[71.5]], [[30.25]], [[34.25]], [[97.0]], [[63.5]], [[37.0]], [[46.5]], [[34.5]], [[65.5]], [[21.25]], [[68.0]], [[55.25]], [[155.0]], [[16.875]], [[89.0]], [[93.0]], [[53.75]], [[28.375]], [[115.0]], [[36.25]], [[59.0]], [[41.75]], [[69.0]], [[70.5]], [[109.0]], [[85.5]], [[36.5]], [[63.5]], [[31.625]], [[64.0]], [[28.125]], [[81.5]], [[27.125]], [[33.25]], [[95.0]], [[68.5]], [[179.0]], [[49.25]], [[64.5]], [[35.5]], [[95.0]], [[61.25]], [[104.5]], [[25.25]], [[132.0]], [[41.0]], [[48.5]], [[42.25]], [[44.25]], [[93.5]], [[47.75]], [[68.5]], [[50.0]], [[98.0]], [[54.0]], [[42.75]], [[23.875]], [[57.75]], [[100.0]], [[55.0]], [[105.0]], [[31.25]], [[53.5]], [[62.25]]]}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.431640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.396484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.341796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[70.0]]], "params": {"weight": [[0.51171875]]}}, "language_model.model.layers.37.feed_forward.shared_expert.down_proj": {"inputs": [[[136.0]]], "outputs": [[[71.0]], [[9.238134570705165e+37]]], "params": {"weight": [[0.78125]]}}, "language_model.model.layers.38.self_attn.qkv_proj": {"inputs": [[[47.25]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.38.self_attn.o_proj": {"inputs": [[[10.0625]]], "outputs": [[[26.75]], [[56.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.38.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.125]], [[18.25]]]}, "language_model.model.layers.38.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.4375]]]}, "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.125]]]}, "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[9.125]]]}, "language_model.model.layers.38.self_attn.attn.impl.k_cache": {"inputs": [[[18.625]]]}, "language_model.model.layers.38.self_attn.attn.impl.v_cache": {"inputs": [[[11.4375]]]}, "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.125]], [[18.625]], [[11.125]]], "outputs": [[[10.0625]], [[1.0]]]}, "language_model.model.layers.38.feed_forward.gate_up_proj": {"inputs": [[[31.125]]], "params": {"weight": [[0.5078125]]}}, "language_model.model.layers.38.feed_forward.down_proj": {"inputs": [[[95.0]]], "outputs": [[[24.75]], [[193.0]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.39.self_attn.qkv_proj": {"inputs": [[[49.75]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.39.self_attn.o_proj": {"inputs": [[[12.3125]]], "outputs": [[[129.0]], [[134.0]]], "params": {"weight": [[1.4140625]]}}, "language_model.model.layers.39.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.703125]], [[7.90625]]]}, "language_model.model.layers.39.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[15.0625]]]}, "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.703125]]]}, "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[11.8125]]]}, "language_model.model.layers.39.self_attn.attn.impl.k_cache": {"inputs": [[[7.90625]]]}, "language_model.model.layers.39.self_attn.attn.impl.v_cache": {"inputs": [[[15.0625]]]}, "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.875]], [[7.90625]], [[14.625]]], "outputs": [[[12.3125]], [[1.0]]]}, "language_model.model.layers.39.feed_forward.router": {"inputs": [[[61.5]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op": {"inputs": [[[61.5]]], "outputs": [[[17.125]], [[84.0]], [[38.0]], [[75.5]], [[72.0]], [[58.0]], [[36.75]], [[33.5]], [[92.0]], [[34.0]], [[47.75]], [[58.25]], [[43.5]], [[110.0]], [[95.5]], [[70.5]], [[42.25]], [[112.5]], [[97.0]], [[45.0]], [[56.0]], [[93.5]], [[117.0]], [[33.75]], [[51.25]], [[122.0]], [[80.0]], [[85.0]], [[97.5]], [[130.0]], [[52.5]], [[56.25]], [[141.0]], [[25.0]], [[49.0]], [[98.0]], [[56.75]], [[81.0]], [[85.5]], [[96.5]], [[59.0]], [[64.0]], [[60.75]], [[58.25]], [[31.375]], [[50.0]], [[40.25]], [[38.25]], [[42.5]], [[77.0]], [[57.0]], [[109.5]], [[114.0]], [[46.5]], [[111.5]], [[63.5]], [[88.0]], [[35.5]], [[104.5]], [[46.75]], [[52.5]], [[118.0]], [[98.0]], [[99.0]], [[27.25]], [[29.625]], [[83.5]], [[35.5]], [[78.0]], [[111.5]], [[79.0]], [[86.0]], [[44.75]], [[96.0]], [[134.0]], [[54.0]], [[71.5]], [[68.5]], [[109.5]], [[131.0]], [[50.75]], [[66.5]], [[78.0]], [[90.0]], [[36.25]], [[113.0]], [[56.75]], [[135.0]], [[46.75]], [[138.0]], [[45.5]], [[46.5]], [[78.0]], [[36.0]], [[50.25]], [[74.0]], [[51.75]], [[97.0]], [[112.0]], [[48.5]], [[38.25]], [[67.0]], [[78.0]], [[113.5]], [[109.0]], [[75.0]], [[125.5]], [[116.5]], [[114.5]], [[70.5]], [[55.75]], [[72.5]], [[49.0]], [[30.125]], [[88.5]], [[83.0]], [[149.0]], [[50.0]], [[29.375]], [[75.5]], [[72.0]], [[29.0]], [[62.0]], [[102.0]], [[50.75]], [[113.5]], [[44.5]], [[95.0]], [[92.5]]]}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.404296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[61.5]]], "params": {"weight": [[0.404296875]]}}, "language_model.model.layers.39.feed_forward.shared_expert.down_proj": {"inputs": [[[126.0]]], "outputs": [[[59.75]], [[9.238134570705165e+37]]], "params": {"weight": [[0.82421875]]}}, "language_model.model.layers.40.self_attn.qkv_proj": {"inputs": [[[46.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.40.self_attn.o_proj": {"inputs": [[[9.125]]], "outputs": [[[93.0]], [[90.0]]], "params": {"weight": [[1.15625]]}}, "language_model.model.layers.40.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0859375]], [[16.125]]]}, "language_model.model.layers.40.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.0625]]]}, "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0859375]]]}, "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[9.125]]]}, "language_model.model.layers.40.self_attn.attn.impl.k_cache": {"inputs": [[[16.125]]]}, "language_model.model.layers.40.self_attn.attn.impl.v_cache": {"inputs": [[[11.0625]]]}, "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.3125]], [[16.125]], [[11.0625]]], "outputs": [[[9.0]], [[1.0]]]}, "language_model.model.layers.40.feed_forward.gate_up_proj": {"inputs": [[[40.75]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.40.feed_forward.down_proj": {"inputs": [[[147.0]]], "outputs": [[[22.625]], [[146.0]]], "params": {"weight": [[0.53515625]]}}, "language_model.model.layers.41.self_attn.qkv_proj": {"inputs": [[[33.75]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.41.self_attn.o_proj": {"inputs": [[[7.84375]]], "outputs": [[[159.0]], [[164.0]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.41.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.46875]], [[11.125]]]}, "language_model.model.layers.41.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.5625]]]}, "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.46875]]]}, "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.96875]]]}, "language_model.model.layers.41.self_attn.attn.impl.k_cache": {"inputs": [[[11.125]]]}, "language_model.model.layers.41.self_attn.attn.impl.v_cache": {"inputs": [[[11.5625]]]}, "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[19.5]], [[11.125]], [[11.5625]]], "outputs": [[[7.84375]], [[1.0]]]}, "language_model.model.layers.41.feed_forward.router": {"inputs": [[[68.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op": {"inputs": [[[68.0]]], "outputs": [[[18.875]], [[77.0]], [[69.0]], [[29.5]], [[97.0]], [[44.25]], [[51.0]], [[180.0]], [[74.0]], [[35.5]], [[68.0]], [[137.0]], [[56.5]], [[74.5]], [[77.5]], [[109.5]], [[95.5]], [[176.0]], [[90.5]], [[147.0]], [[34.75]], [[108.5]], [[39.75]], [[90.5]], [[100.0]], [[114.0]], [[150.0]], [[129.0]], [[66.5]], [[94.5]], [[46.25]], [[68.0]], [[113.0]], [[120.5]], [[58.0]], [[120.0]], [[89.5]], [[93.0]], [[159.0]], [[84.5]], [[60.25]], [[65.0]], [[128.0]], [[84.0]], [[42.0]], [[65.0]], [[75.0]], [[92.0]], [[71.0]], [[184.0]], [[109.0]], [[160.0]], [[82.5]], [[79.5]], [[96.0]], [[91.0]], [[80.0]], [[142.0]], [[69.5]], [[76.5]], [[45.75]], [[172.0]], [[88.0]], [[75.0]], [[89.0]], [[74.0]], [[93.0]], [[111.0]], [[68.0]], [[31.125]], [[94.0]], [[46.75]], [[35.25]], [[88.5]], [[38.5]], [[87.5]], [[129.0]], [[84.0]], [[45.0]], [[72.0]], [[103.0]], [[115.0]], [[129.0]], [[76.5]], [[73.0]], [[80.0]], [[55.5]], [[61.5]], [[47.5]], [[60.5]], [[109.0]], [[82.0]], [[67.5]], [[251.0]], [[91.5]], [[62.5]], [[64.0]], [[45.75]], [[70.5]], [[58.25]], [[77.0]], [[126.0]], [[162.0]], [[92.0]], [[97.0]], [[78.5]], [[133.0]], [[59.5]], [[87.0]], [[34.0]], [[76.5]], [[154.0]], [[132.0]], [[83.5]], [[69.5]], [[56.5]], [[146.0]], [[48.25]], [[174.0]], [[138.0]], [[112.0]], [[95.5]], [[64.5]], [[138.0]], [[39.25]], [[76.5]], [[64.0]], [[58.75]], [[124.0]]]}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.4453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.39453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[68.0]]], "params": {"weight": [[0.4609375]]}}, "language_model.model.layers.41.feed_forward.shared_expert.down_proj": {"inputs": [[[216.0]]], "outputs": [[[26.125]], [[9.238134570705165e+37]]], "params": {"weight": [[0.625]]}}, "language_model.model.layers.42.self_attn.qkv_proj": {"inputs": [[[38.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.42.self_attn.o_proj": {"inputs": [[[12.75]]], "outputs": [[[50.5]], [[134.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.42.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.3671875]], [[10.9375]]]}, "language_model.model.layers.42.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[17.25]]]}, "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.3671875]]]}, "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[12.75]]]}, "language_model.model.layers.42.self_attn.attn.impl.k_cache": {"inputs": [[[10.9375]]]}, "language_model.model.layers.42.self_attn.attn.impl.v_cache": {"inputs": [[[17.25]]]}, "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[16.875]], [[10.9375]], [[17.25]]], "outputs": [[[11.6875]], [[1.0]]]}, "language_model.model.layers.42.feed_forward.gate_up_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.42.feed_forward.down_proj": {"inputs": [[[150.0]]], "outputs": [[[59.5]], [[338.0]]], "params": {"weight": [[0.41015625]]}}, "language_model.model.layers.43.self_attn.qkv_proj": {"inputs": [[[45.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.43.self_attn.o_proj": {"inputs": [[[11.875]]], "outputs": [[[94.5]], [[232.0]]], "params": {"weight": [[1.0625]]}}, "language_model.model.layers.43.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.140625]], [[10.9375]]]}, "language_model.model.layers.43.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.1875]]]}, "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.140625]]]}, "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[9.3125]]]}, "language_model.model.layers.43.self_attn.attn.impl.k_cache": {"inputs": [[[10.9375]]]}, "language_model.model.layers.43.self_attn.attn.impl.v_cache": {"inputs": [[[12.1875]]]}, "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.0625]], [[10.9375]], [[12.1875]]], "outputs": [[[11.875]], [[1.0]]]}, "language_model.model.layers.43.feed_forward.router": {"inputs": [[[68.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op": {"inputs": [[[68.0]]], "outputs": [[[27.25]], [[83.0]], [[86.0]], [[84.0]], [[103.5]], [[208.0]], [[48.75]], [[55.75]], [[121.0]], [[148.0]], [[108.5]], [[72.0]], [[138.0]], [[159.0]], [[116.5]], [[174.0]], [[184.0]], [[101.0]], [[53.0]], [[206.0]], [[40.25]], [[179.0]], [[131.0]], [[112.0]], [[120.0]], [[96.5]], [[103.0]], [[82.5]], [[66.0]], [[119.5]], [[81.5]], [[68.5]], [[49.0]], [[44.0]], [[123.5]], [[90.0]], [[129.0]], [[79.0]], [[86.0]], [[68.5]], [[180.0]], [[90.5]], [[204.0]], [[181.0]], [[41.0]], [[206.0]], [[74.5]], [[71.5]], [[41.0]], [[69.0]], [[63.75]], [[49.25]], [[166.0]], [[143.0]], [[67.5]], [[202.0]], [[102.5]], [[36.5]], [[36.0]], [[102.0]], [[88.0]], [[67.5]], [[55.75]], [[82.0]], [[164.0]], [[163.0]], [[142.0]], [[97.5]], [[117.5]], [[199.0]], [[63.5]], [[89.0]], [[80.5]], [[57.0]], [[153.0]], [[37.0]], [[78.0]], [[145.0]], [[40.5]], [[142.0]], [[82.0]], [[135.0]], [[132.0]], [[111.0]], [[74.0]], [[111.5]], [[76.0]], [[62.0]], [[82.5]], [[70.5]], [[133.0]], [[154.0]], [[62.25]], [[87.5]], [[60.5]], [[40.0]], [[50.75]], [[36.25]], [[132.0]], [[58.75]], [[77.5]], [[92.0]], [[77.5]], [[64.5]], [[75.0]], [[94.0]], [[110.0]], [[160.0]], [[106.0]], [[50.75]], [[99.0]], [[163.0]], [[76.5]], [[132.0]], [[77.0]], [[39.25]], [[210.0]], [[68.5]], [[144.0]], [[78.5]], [[40.75]], [[67.0]], [[94.5]], [[108.0]], [[101.0]], [[86.0]], [[178.0]], [[96.5]], [[85.5]]]}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.4140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.3828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.34765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[68.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.43.feed_forward.shared_expert.down_proj": {"inputs": [[[222.0]]], "outputs": [[[25.125]], [[9.238134570705165e+37]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.44.self_attn.qkv_proj": {"inputs": [[[28.625]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.44.self_attn.o_proj": {"inputs": [[[13.8125]]], "outputs": [[[68.0]], [[153.0]]], "params": {"weight": [[0.8828125]]}}, "language_model.model.layers.44.self_attn.attn.impl.matmul_qk": {"inputs": [[[2.03125]], [[18.0]]]}, "language_model.model.layers.44.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[14.1875]]]}, "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[2.03125]]]}, "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[10.5]]]}, "language_model.model.layers.44.self_attn.attn.impl.k_cache": {"inputs": [[[18.0]]]}, "language_model.model.layers.44.self_attn.attn.impl.v_cache": {"inputs": [[[14.1875]]]}, "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[24.5]], [[18.0]], [[14.1875]]], "outputs": [[[13.8125]], [[1.0]]]}, "language_model.model.layers.44.feed_forward.gate_up_proj": {"inputs": [[[41.5]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.44.feed_forward.down_proj": {"inputs": [[[228.0]]], "outputs": [[[63.25]], [[191.0]]], "params": {"weight": [[0.419921875]]}}, "language_model.model.layers.45.self_attn.qkv_proj": {"inputs": [[[27.375]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.45.self_attn.o_proj": {"inputs": [[[24.25]]], "outputs": [[[169.0]], [[220.0]]], "params": {"weight": [[1.1640625]]}}, "language_model.model.layers.45.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.65625]], [[14.4375]]]}, "language_model.model.layers.45.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[27.75]]]}, "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.65625]]]}, "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[14.6875]]]}, "language_model.model.layers.45.self_attn.attn.impl.k_cache": {"inputs": [[[14.4375]]]}, "language_model.model.layers.45.self_attn.attn.impl.v_cache": {"inputs": [[[27.75]]]}, "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[20.75]], [[14.4375]], [[27.75]]], "outputs": [[[24.25]], [[1.0]]]}, "language_model.model.layers.45.feed_forward.router": {"inputs": [[[51.75]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op": {"inputs": [[[51.75]]], "outputs": [[[10.5625]], [[41.25]], [[19.375]], [[47.25]], [[55.0]], [[0.0]], [[28.375]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[28.0]], [[36.0]], [[56.75]], [[66.5]], [[0.0]], [[0.0]], [[0.0]], [[38.75]], [[0.0]], [[0.0]], [[0.0]], [[29.875]], [[0.0]], [[25.125]], [[39.25]], [[107.5]], [[57.25]], [[43.5]], [[0.0]], [[33.25]], [[19.25]], [[55.0]], [[93.5]], [[83.0]], [[34.0]], [[0.0]], [[30.375]], [[23.625]], [[61.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[69.0]], [[0.0]], [[65.5]], [[0.0]], [[0.0]], [[0.0]], [[40.5]], [[0.0]], [[86.5]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[42.5]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[48.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[54.75]], [[26.375]], [[84.0]], [[22.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[48.25]], [[0.0]], [[41.0]], [[39.25]], [[71.0]], [[69.0]], [[0.0]], [[0.0]], [[27.875]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[18.25]], [[5.6107462426124854e-18]], [[52.75]], [[0.0]], [[96.0]], [[0.0]], [[0.0]]]}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.380859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[51.75]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.45.feed_forward.shared_expert.down_proj": {"inputs": [[[354.0]]], "outputs": [[[33.75]], [[9.238134570705165e+37]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.46.self_attn.qkv_proj": {"inputs": [[[26.75]]], "params": {"weight": [[0.32421875]]}}, "language_model.model.layers.46.self_attn.o_proj": {"inputs": [[[21.375]]], "outputs": [[[101.0]], [[188.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.46.self_attn.attn.impl.matmul_qk": {"inputs": [[[2.703125]], [[14.9375]]]}, "language_model.model.layers.46.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[21.375]]]}, "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[2.703125]]]}, "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[21.375]]]}, "language_model.model.layers.46.self_attn.attn.impl.k_cache": {"inputs": [[[14.9375]]]}, "language_model.model.layers.46.self_attn.attn.impl.v_cache": {"inputs": [[[21.375]]]}, "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[28.0]], [[14.9375]], [[18.375]]], "outputs": [[[18.375]], [[1.0]]]}, "language_model.model.layers.46.feed_forward.gate_up_proj": {"inputs": [[[40.5]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.46.feed_forward.down_proj": {"inputs": [[[438.0]]], "outputs": [[[1688.0]], [[2400.0]]], "params": {"weight": [[0.65234375]]}}, "language_model.model.layers.47.self_attn.qkv_proj": {"inputs": [[[19.875]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.47.self_attn.o_proj": {"inputs": [[[5.75]]], "outputs": [[[98.0]], [[126.5]]], "params": {"weight": [[0.408203125]]}}, "language_model.model.layers.47.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.474609375]], [[6.75]]]}, "language_model.model.layers.47.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.75]]]}, "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.474609375]]]}, "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.875]]]}, "language_model.model.layers.47.self_attn.attn.impl.k_cache": {"inputs": [[[6.75]]]}, "language_model.model.layers.47.self_attn.attn.impl.v_cache": {"inputs": [[[9.75]]]}, "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[4.875]], [[5.9375]], [[9.75]]], "outputs": [[[5.75]], [[1.0]]]}, "language_model.model.layers.47.feed_forward.router": {"inputs": [[[33.5]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op": {"inputs": [[[33.5]]], "outputs": [[[185.0]], [[58.75]], [[124.0]], [[64.5]], [[44.0]], [[95.5]], [[54.25]], [[51.0]], [[35.0]], [[124.5]], [[264.0]], [[61.25]], [[102.0]], [[106.0]], [[40.75]], [[128.0]], [[30.0]], [[115.5]], [[90.0]], [[52.25]], [[72.0]], [[358.0]], [[86.0]], [[41.75]], [[64.5]], [[225.0]], [[292.0]], [[27.5]], [[26.75]], [[118.0]], [[60.25]], [[43.5]], [[124.0]], [[23.5]], [[35.75]], [[136.0]], [[57.5]], [[26.75]], [[22.375]], [[211.0]], [[59.75]], [[33.0]], [[97.5]], [[246.0]], [[36.25]], [[39.75]], [[79.0]], [[165.0]], [[43.5]], [[22.0]], [[110.0]], [[177.0]], [[340.0]], [[57.75]], [[34.25]], [[36.75]], [[26.25]], [[101.0]], [[41.75]], [[40.25]], [[90.0]], [[42.25]], [[91.0]], [[93.0]], [[206.0]], [[39.75]], [[29.25]], [[25.5]], [[165.0]], [[27.5]], [[129.0]], [[290.0]], [[37.0]], [[26.375]], [[23.75]], [[71.0]], [[31.25]], [[43.0]], [[52.0]], [[149.0]], [[35.0]], [[74.0]], [[258.0]], [[488.0]], [[43.0]], [[49.75]], [[118.0]], [[274.0]], [[37.5]], [[35.25]], [[141.0]], [[101.0]], [[127.5]], [[386.0]], [[37.25]], [[36.25]], [[118.5]], [[98.0]], [[31.25]], [[30.75]], [[62.25]], [[72.5]], [[64.5]], [[96.0]], [[62.0]], [[51.0]], [[31.875]], [[46.75]], [[324.0]], [[41.0]], [[59.0]], [[221.0]], [[157.0]], [[52.25]], [[71.0]], [[128.0]], [[65.0]], [[59.25]], [[202.0]], [[32.5]], [[224.0]], [[219.0]], [[140.0]], [[73.5]], [[24.625]], [[68.5]], [[152.0]], [[144.0]], [[316.0]]]}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.494140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.84765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.32421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.5546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.73046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.46484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.470703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.376953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.470703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.61328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.8125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.42578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.39453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.427734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.48828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.78125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.5]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.62890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.37109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.365234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.8125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.7265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.470703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.6171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.7109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.6328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.7265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.361328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.55078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.36328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.35546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.75]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.45703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.42578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.482421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.466796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.400390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.66015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.6640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.41015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.6640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.8046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.44921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.431640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.73828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.36328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.66796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.416015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.341796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.59375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.6875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.75]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.7421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.8359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.796875]]}}, "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[33.5]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.47.feed_forward.shared_expert.down_proj": {"inputs": [[[676.0]]], "outputs": [[[592.0]], [[9.238134570705165e+37]]], "params": {"weight": [[0.416015625]]}}, "language_model.lm_head": {"inputs": [[[52.0]]], "params": {"weight": [[1.1875]]}}}} \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_1_8.npz b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_1_8.npz new file mode 100644 index 000000000000..a6c50ce437c8 Binary files /dev/null and b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_1_8.npz differ diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_1_8_mod_list.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_1_8_mod_list.json new file mode 100644 index 000000000000..928cd9e2a2c5 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_1_8_mod_list.json @@ -0,0 +1,6935 @@ +[ + "vision_model.patch_embedding.linear", + "vision_model.model.layers.0.self_attn.qkv_proj", + "vision_model.model.layers.0.self_attn.o_proj", + "vision_model.model.layers.0.mlp.fc1", + "vision_model.model.layers.0.mlp.fc2", + "vision_model.model.layers.1.self_attn.qkv_proj", + "vision_model.model.layers.1.self_attn.o_proj", + "vision_model.model.layers.1.mlp.fc1", + "vision_model.model.layers.1.mlp.fc2", + "vision_model.model.layers.2.self_attn.qkv_proj", + "vision_model.model.layers.2.self_attn.o_proj", + "vision_model.model.layers.2.mlp.fc1", + "vision_model.model.layers.2.mlp.fc2", + "vision_model.model.layers.3.self_attn.qkv_proj", + "vision_model.model.layers.3.self_attn.o_proj", + "vision_model.model.layers.3.mlp.fc1", + "vision_model.model.layers.3.mlp.fc2", + "vision_model.model.layers.4.self_attn.qkv_proj", + "vision_model.model.layers.4.self_attn.o_proj", + "vision_model.model.layers.4.mlp.fc1", + "vision_model.model.layers.4.mlp.fc2", + "vision_model.model.layers.5.self_attn.qkv_proj", + "vision_model.model.layers.5.self_attn.o_proj", + "vision_model.model.layers.5.mlp.fc1", + "vision_model.model.layers.5.mlp.fc2", + "vision_model.model.layers.6.self_attn.qkv_proj", + "vision_model.model.layers.6.self_attn.o_proj", + "vision_model.model.layers.6.mlp.fc1", + "vision_model.model.layers.6.mlp.fc2", + "vision_model.model.layers.7.self_attn.qkv_proj", + "vision_model.model.layers.7.self_attn.o_proj", + "vision_model.model.layers.7.mlp.fc1", + "vision_model.model.layers.7.mlp.fc2", + "vision_model.model.layers.8.self_attn.qkv_proj", + "vision_model.model.layers.8.self_attn.o_proj", + "vision_model.model.layers.8.mlp.fc1", + "vision_model.model.layers.8.mlp.fc2", + "vision_model.model.layers.9.self_attn.qkv_proj", + "vision_model.model.layers.9.self_attn.o_proj", + "vision_model.model.layers.9.mlp.fc1", + "vision_model.model.layers.9.mlp.fc2", + "vision_model.model.layers.10.self_attn.qkv_proj", + "vision_model.model.layers.10.self_attn.o_proj", + "vision_model.model.layers.10.mlp.fc1", + "vision_model.model.layers.10.mlp.fc2", + "vision_model.model.layers.11.self_attn.qkv_proj", + "vision_model.model.layers.11.self_attn.o_proj", + "vision_model.model.layers.11.mlp.fc1", + "vision_model.model.layers.11.mlp.fc2", + "vision_model.model.layers.12.self_attn.qkv_proj", + "vision_model.model.layers.12.self_attn.o_proj", + "vision_model.model.layers.12.mlp.fc1", + "vision_model.model.layers.12.mlp.fc2", + "vision_model.model.layers.13.self_attn.qkv_proj", + "vision_model.model.layers.13.self_attn.o_proj", + "vision_model.model.layers.13.mlp.fc1", + "vision_model.model.layers.13.mlp.fc2", + "vision_model.model.layers.14.self_attn.qkv_proj", + "vision_model.model.layers.14.self_attn.o_proj", + "vision_model.model.layers.14.mlp.fc1", + "vision_model.model.layers.14.mlp.fc2", + "vision_model.model.layers.15.self_attn.qkv_proj", + "vision_model.model.layers.15.self_attn.o_proj", + "vision_model.model.layers.15.mlp.fc1", + "vision_model.model.layers.15.mlp.fc2", + "vision_model.model.layers.16.self_attn.qkv_proj", + "vision_model.model.layers.16.self_attn.o_proj", + "vision_model.model.layers.16.mlp.fc1", + "vision_model.model.layers.16.mlp.fc2", + "vision_model.model.layers.17.self_attn.qkv_proj", + "vision_model.model.layers.17.self_attn.o_proj", + "vision_model.model.layers.17.mlp.fc1", + "vision_model.model.layers.17.mlp.fc2", + "vision_model.model.layers.18.self_attn.qkv_proj", + "vision_model.model.layers.18.self_attn.o_proj", + "vision_model.model.layers.18.mlp.fc1", + "vision_model.model.layers.18.mlp.fc2", + "vision_model.model.layers.19.self_attn.qkv_proj", + "vision_model.model.layers.19.self_attn.o_proj", + "vision_model.model.layers.19.mlp.fc1", + "vision_model.model.layers.19.mlp.fc2", + "vision_model.model.layers.20.self_attn.qkv_proj", + "vision_model.model.layers.20.self_attn.o_proj", + "vision_model.model.layers.20.mlp.fc1", + "vision_model.model.layers.20.mlp.fc2", + "vision_model.model.layers.21.self_attn.qkv_proj", + "vision_model.model.layers.21.self_attn.o_proj", + "vision_model.model.layers.21.mlp.fc1", + "vision_model.model.layers.21.mlp.fc2", + "vision_model.model.layers.22.self_attn.qkv_proj", + "vision_model.model.layers.22.self_attn.o_proj", + "vision_model.model.layers.22.mlp.fc1", + "vision_model.model.layers.22.mlp.fc2", + "vision_model.model.layers.23.self_attn.qkv_proj", + "vision_model.model.layers.23.self_attn.o_proj", + "vision_model.model.layers.23.mlp.fc1", + "vision_model.model.layers.23.mlp.fc2", + "vision_model.model.layers.24.self_attn.qkv_proj", + "vision_model.model.layers.24.self_attn.o_proj", + "vision_model.model.layers.24.mlp.fc1", + "vision_model.model.layers.24.mlp.fc2", + "vision_model.model.layers.25.self_attn.qkv_proj", + "vision_model.model.layers.25.self_attn.o_proj", + "vision_model.model.layers.25.mlp.fc1", + "vision_model.model.layers.25.mlp.fc2", + "vision_model.model.layers.26.self_attn.qkv_proj", + "vision_model.model.layers.26.self_attn.o_proj", + "vision_model.model.layers.26.mlp.fc1", + "vision_model.model.layers.26.mlp.fc2", + "vision_model.model.layers.27.self_attn.qkv_proj", + "vision_model.model.layers.27.self_attn.o_proj", + "vision_model.model.layers.27.mlp.fc1", + "vision_model.model.layers.27.mlp.fc2", + "vision_model.model.layers.28.self_attn.qkv_proj", + "vision_model.model.layers.28.self_attn.o_proj", + "vision_model.model.layers.28.mlp.fc1", + "vision_model.model.layers.28.mlp.fc2", + "vision_model.model.layers.29.self_attn.qkv_proj", + "vision_model.model.layers.29.self_attn.o_proj", + "vision_model.model.layers.29.mlp.fc1", + "vision_model.model.layers.29.mlp.fc2", + "vision_model.model.layers.30.self_attn.qkv_proj", + "vision_model.model.layers.30.self_attn.o_proj", + "vision_model.model.layers.30.mlp.fc1", + "vision_model.model.layers.30.mlp.fc2", + "vision_model.model.layers.31.self_attn.qkv_proj", + "vision_model.model.layers.31.self_attn.o_proj", + "vision_model.model.layers.31.mlp.fc1", + "vision_model.model.layers.31.mlp.fc2", + "vision_model.model.layers.32.self_attn.qkv_proj", + "vision_model.model.layers.32.self_attn.o_proj", + "vision_model.model.layers.32.mlp.fc1", + "vision_model.model.layers.32.mlp.fc2", + "vision_model.model.layers.33.self_attn.qkv_proj", + "vision_model.model.layers.33.self_attn.o_proj", + "vision_model.model.layers.33.mlp.fc1", + "vision_model.model.layers.33.mlp.fc2", + "vision_model.vision_adapter.mlp.fc1", + "vision_model.vision_adapter.mlp.fc2", + "multi_modal_projector.linear_1", + "language_model.model.layers.0.self_attn.qkv_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.0.self_attn.attn.impl.softmax", + "language_model.model.layers.0.self_attn.attn.impl.matmul_av", + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.0.self_attn.attn.impl.k_cache", + "language_model.model.layers.0.self_attn.attn.impl.v_cache", + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.0.feed_forward.gate_up_proj", + "language_model.model.layers.0.feed_forward.down_proj", + "language_model.model.layers.1.self_attn.qkv_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.1.self_attn.attn.impl.softmax", + "language_model.model.layers.1.self_attn.attn.impl.matmul_av", + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.1.self_attn.attn.impl.k_cache", + "language_model.model.layers.1.self_attn.attn.impl.v_cache", + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.1.feed_forward.router", + "language_model.model.layers.1.feed_forward.experts", + "language_model.model.layers.1.feed_forward.experts.moe_op", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.1.feed_forward.shared_expert.down_proj", + "language_model.model.layers.2.self_attn.qkv_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.2.self_attn.attn.impl.softmax", + "language_model.model.layers.2.self_attn.attn.impl.matmul_av", + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.2.self_attn.attn.impl.k_cache", + "language_model.model.layers.2.self_attn.attn.impl.v_cache", + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.2.feed_forward.gate_up_proj", + "language_model.model.layers.2.feed_forward.down_proj", + "language_model.model.layers.3.self_attn.qkv_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.3.self_attn.attn.impl.softmax", + "language_model.model.layers.3.self_attn.attn.impl.matmul_av", + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.3.self_attn.attn.impl.k_cache", + "language_model.model.layers.3.self_attn.attn.impl.v_cache", + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.3.feed_forward.router", + "language_model.model.layers.3.feed_forward.experts", + "language_model.model.layers.3.feed_forward.experts.moe_op", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.3.feed_forward.shared_expert.down_proj", + "language_model.model.layers.4.self_attn.qkv_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.4.self_attn.attn.impl.softmax", + "language_model.model.layers.4.self_attn.attn.impl.matmul_av", + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.4.self_attn.attn.impl.k_cache", + "language_model.model.layers.4.self_attn.attn.impl.v_cache", + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.4.feed_forward.gate_up_proj", + "language_model.model.layers.4.feed_forward.down_proj", + "language_model.model.layers.5.self_attn.qkv_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.5.self_attn.attn.impl.softmax", + "language_model.model.layers.5.self_attn.attn.impl.matmul_av", + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.5.self_attn.attn.impl.k_cache", + "language_model.model.layers.5.self_attn.attn.impl.v_cache", + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.5.feed_forward.router", + "language_model.model.layers.5.feed_forward.experts", + "language_model.model.layers.5.feed_forward.experts.moe_op", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.5.feed_forward.shared_expert.down_proj", + "language_model.model.layers.6.self_attn.qkv_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.6.self_attn.attn.impl.softmax", + "language_model.model.layers.6.self_attn.attn.impl.matmul_av", + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.6.self_attn.attn.impl.k_cache", + "language_model.model.layers.6.self_attn.attn.impl.v_cache", + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.6.feed_forward.gate_up_proj", + "language_model.model.layers.6.feed_forward.down_proj", + "language_model.model.layers.7.self_attn.qkv_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.7.self_attn.attn.impl.softmax", + "language_model.model.layers.7.self_attn.attn.impl.matmul_av", + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.7.self_attn.attn.impl.k_cache", + "language_model.model.layers.7.self_attn.attn.impl.v_cache", + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.7.feed_forward.router", + "language_model.model.layers.7.feed_forward.experts", + "language_model.model.layers.7.feed_forward.experts.moe_op", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.7.feed_forward.shared_expert.down_proj", + "language_model.model.layers.8.self_attn.qkv_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.8.self_attn.attn.impl.softmax", + "language_model.model.layers.8.self_attn.attn.impl.matmul_av", + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.8.self_attn.attn.impl.k_cache", + "language_model.model.layers.8.self_attn.attn.impl.v_cache", + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.8.feed_forward.gate_up_proj", + "language_model.model.layers.8.feed_forward.down_proj", + "language_model.model.layers.9.self_attn.qkv_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.9.self_attn.attn.impl.softmax", + "language_model.model.layers.9.self_attn.attn.impl.matmul_av", + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.9.self_attn.attn.impl.k_cache", + "language_model.model.layers.9.self_attn.attn.impl.v_cache", + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.9.feed_forward.router", + "language_model.model.layers.9.feed_forward.experts", + "language_model.model.layers.9.feed_forward.experts.moe_op", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.9.feed_forward.shared_expert.down_proj", + "language_model.model.layers.10.self_attn.qkv_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.10.self_attn.attn.impl.softmax", + "language_model.model.layers.10.self_attn.attn.impl.matmul_av", + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.10.self_attn.attn.impl.k_cache", + "language_model.model.layers.10.self_attn.attn.impl.v_cache", + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.10.feed_forward.gate_up_proj", + "language_model.model.layers.10.feed_forward.down_proj", + "language_model.model.layers.11.self_attn.qkv_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.11.self_attn.attn.impl.softmax", + "language_model.model.layers.11.self_attn.attn.impl.matmul_av", + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.11.self_attn.attn.impl.k_cache", + "language_model.model.layers.11.self_attn.attn.impl.v_cache", + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.11.feed_forward.router", + "language_model.model.layers.11.feed_forward.experts", + "language_model.model.layers.11.feed_forward.experts.moe_op", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.11.feed_forward.shared_expert.down_proj", + "language_model.model.layers.12.self_attn.qkv_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.12.self_attn.attn.impl.softmax", + "language_model.model.layers.12.self_attn.attn.impl.matmul_av", + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.12.self_attn.attn.impl.k_cache", + "language_model.model.layers.12.self_attn.attn.impl.v_cache", + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.12.feed_forward.gate_up_proj", + "language_model.model.layers.12.feed_forward.down_proj", + "language_model.model.layers.13.self_attn.qkv_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.13.self_attn.attn.impl.softmax", + "language_model.model.layers.13.self_attn.attn.impl.matmul_av", + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.13.self_attn.attn.impl.k_cache", + "language_model.model.layers.13.self_attn.attn.impl.v_cache", + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.13.feed_forward.router", + "language_model.model.layers.13.feed_forward.experts", + "language_model.model.layers.13.feed_forward.experts.moe_op", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.13.feed_forward.shared_expert.down_proj", + "language_model.model.layers.14.self_attn.qkv_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.14.self_attn.attn.impl.softmax", + "language_model.model.layers.14.self_attn.attn.impl.matmul_av", + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.14.self_attn.attn.impl.k_cache", + "language_model.model.layers.14.self_attn.attn.impl.v_cache", + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.14.feed_forward.gate_up_proj", + "language_model.model.layers.14.feed_forward.down_proj", + "language_model.model.layers.15.self_attn.qkv_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.15.self_attn.attn.impl.softmax", + "language_model.model.layers.15.self_attn.attn.impl.matmul_av", + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.15.self_attn.attn.impl.k_cache", + "language_model.model.layers.15.self_attn.attn.impl.v_cache", + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.15.feed_forward.router", + "language_model.model.layers.15.feed_forward.experts", + "language_model.model.layers.15.feed_forward.experts.moe_op", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.15.feed_forward.shared_expert.down_proj", + "language_model.model.layers.16.self_attn.qkv_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.16.self_attn.attn.impl.softmax", + "language_model.model.layers.16.self_attn.attn.impl.matmul_av", + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.16.self_attn.attn.impl.k_cache", + "language_model.model.layers.16.self_attn.attn.impl.v_cache", + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.16.feed_forward.gate_up_proj", + "language_model.model.layers.16.feed_forward.down_proj", + "language_model.model.layers.17.self_attn.qkv_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.17.self_attn.attn.impl.softmax", + "language_model.model.layers.17.self_attn.attn.impl.matmul_av", + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.17.self_attn.attn.impl.k_cache", + "language_model.model.layers.17.self_attn.attn.impl.v_cache", + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.17.feed_forward.router", + "language_model.model.layers.17.feed_forward.experts", + "language_model.model.layers.17.feed_forward.experts.moe_op", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.17.feed_forward.shared_expert.down_proj", + "language_model.model.layers.18.self_attn.qkv_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.18.self_attn.attn.impl.softmax", + "language_model.model.layers.18.self_attn.attn.impl.matmul_av", + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.18.self_attn.attn.impl.k_cache", + "language_model.model.layers.18.self_attn.attn.impl.v_cache", + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.18.feed_forward.gate_up_proj", + "language_model.model.layers.18.feed_forward.down_proj", + "language_model.model.layers.19.self_attn.qkv_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.19.self_attn.attn.impl.softmax", + "language_model.model.layers.19.self_attn.attn.impl.matmul_av", + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.19.self_attn.attn.impl.k_cache", + "language_model.model.layers.19.self_attn.attn.impl.v_cache", + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.19.feed_forward.router", + "language_model.model.layers.19.feed_forward.experts", + "language_model.model.layers.19.feed_forward.experts.moe_op", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.19.feed_forward.shared_expert.down_proj", + "language_model.model.layers.20.self_attn.qkv_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.20.self_attn.attn.impl.softmax", + "language_model.model.layers.20.self_attn.attn.impl.matmul_av", + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.20.self_attn.attn.impl.k_cache", + "language_model.model.layers.20.self_attn.attn.impl.v_cache", + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.20.feed_forward.gate_up_proj", + "language_model.model.layers.20.feed_forward.down_proj", + "language_model.model.layers.21.self_attn.qkv_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.21.self_attn.attn.impl.softmax", + "language_model.model.layers.21.self_attn.attn.impl.matmul_av", + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.21.self_attn.attn.impl.k_cache", + "language_model.model.layers.21.self_attn.attn.impl.v_cache", + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.21.feed_forward.router", + "language_model.model.layers.21.feed_forward.experts", + "language_model.model.layers.21.feed_forward.experts.moe_op", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.21.feed_forward.shared_expert.down_proj", + "language_model.model.layers.22.self_attn.qkv_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.22.self_attn.attn.impl.softmax", + "language_model.model.layers.22.self_attn.attn.impl.matmul_av", + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.22.self_attn.attn.impl.k_cache", + "language_model.model.layers.22.self_attn.attn.impl.v_cache", + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.22.feed_forward.gate_up_proj", + "language_model.model.layers.22.feed_forward.down_proj", + "language_model.model.layers.23.self_attn.qkv_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.23.self_attn.attn.impl.softmax", + "language_model.model.layers.23.self_attn.attn.impl.matmul_av", + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.23.self_attn.attn.impl.k_cache", + "language_model.model.layers.23.self_attn.attn.impl.v_cache", + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.23.feed_forward.router", + "language_model.model.layers.23.feed_forward.experts", + "language_model.model.layers.23.feed_forward.experts.moe_op", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.23.feed_forward.shared_expert.down_proj", + "language_model.model.layers.24.self_attn.qkv_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.24.self_attn.attn.impl.softmax", + "language_model.model.layers.24.self_attn.attn.impl.matmul_av", + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.24.self_attn.attn.impl.k_cache", + "language_model.model.layers.24.self_attn.attn.impl.v_cache", + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.24.feed_forward.gate_up_proj", + "language_model.model.layers.24.feed_forward.down_proj", + "language_model.model.layers.25.self_attn.qkv_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.25.self_attn.attn.impl.softmax", + "language_model.model.layers.25.self_attn.attn.impl.matmul_av", + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.25.self_attn.attn.impl.k_cache", + "language_model.model.layers.25.self_attn.attn.impl.v_cache", + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.25.feed_forward.router", + "language_model.model.layers.25.feed_forward.experts", + "language_model.model.layers.25.feed_forward.experts.moe_op", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.25.feed_forward.shared_expert.down_proj", + "language_model.model.layers.26.self_attn.qkv_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.26.self_attn.attn.impl.softmax", + "language_model.model.layers.26.self_attn.attn.impl.matmul_av", + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.26.self_attn.attn.impl.k_cache", + "language_model.model.layers.26.self_attn.attn.impl.v_cache", + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.26.feed_forward.gate_up_proj", + "language_model.model.layers.26.feed_forward.down_proj", + "language_model.model.layers.27.self_attn.qkv_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.27.self_attn.attn.impl.softmax", + "language_model.model.layers.27.self_attn.attn.impl.matmul_av", + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.27.self_attn.attn.impl.k_cache", + "language_model.model.layers.27.self_attn.attn.impl.v_cache", + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.27.feed_forward.router", + "language_model.model.layers.27.feed_forward.experts", + "language_model.model.layers.27.feed_forward.experts.moe_op", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.27.feed_forward.shared_expert.down_proj", + "language_model.model.layers.28.self_attn.qkv_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.28.self_attn.attn.impl.softmax", + "language_model.model.layers.28.self_attn.attn.impl.matmul_av", + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.28.self_attn.attn.impl.k_cache", + "language_model.model.layers.28.self_attn.attn.impl.v_cache", + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.28.feed_forward.gate_up_proj", + "language_model.model.layers.28.feed_forward.down_proj", + "language_model.model.layers.29.self_attn.qkv_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.29.self_attn.attn.impl.softmax", + "language_model.model.layers.29.self_attn.attn.impl.matmul_av", + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.29.self_attn.attn.impl.k_cache", + "language_model.model.layers.29.self_attn.attn.impl.v_cache", + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.29.feed_forward.router", + "language_model.model.layers.29.feed_forward.experts", + "language_model.model.layers.29.feed_forward.experts.moe_op", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.29.feed_forward.shared_expert.down_proj", + "language_model.model.layers.30.self_attn.qkv_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.30.self_attn.attn.impl.softmax", + "language_model.model.layers.30.self_attn.attn.impl.matmul_av", + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.30.self_attn.attn.impl.k_cache", + "language_model.model.layers.30.self_attn.attn.impl.v_cache", + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.30.feed_forward.gate_up_proj", + "language_model.model.layers.30.feed_forward.down_proj", + "language_model.model.layers.31.self_attn.qkv_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.31.self_attn.attn.impl.softmax", + "language_model.model.layers.31.self_attn.attn.impl.matmul_av", + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.31.self_attn.attn.impl.k_cache", + "language_model.model.layers.31.self_attn.attn.impl.v_cache", + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.31.feed_forward.router", + "language_model.model.layers.31.feed_forward.experts", + "language_model.model.layers.31.feed_forward.experts.moe_op", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.31.feed_forward.shared_expert.down_proj", + "language_model.model.layers.32.self_attn.qkv_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.32.self_attn.attn.impl.softmax", + "language_model.model.layers.32.self_attn.attn.impl.matmul_av", + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.32.self_attn.attn.impl.k_cache", + "language_model.model.layers.32.self_attn.attn.impl.v_cache", + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.32.feed_forward.gate_up_proj", + "language_model.model.layers.32.feed_forward.down_proj", + "language_model.model.layers.33.self_attn.qkv_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.33.self_attn.attn.impl.softmax", + "language_model.model.layers.33.self_attn.attn.impl.matmul_av", + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.33.self_attn.attn.impl.k_cache", + "language_model.model.layers.33.self_attn.attn.impl.v_cache", + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.33.feed_forward.router", + "language_model.model.layers.33.feed_forward.experts", + "language_model.model.layers.33.feed_forward.experts.moe_op", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.33.feed_forward.shared_expert.down_proj", + "language_model.model.layers.34.self_attn.qkv_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.34.self_attn.attn.impl.softmax", + "language_model.model.layers.34.self_attn.attn.impl.matmul_av", + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.34.self_attn.attn.impl.k_cache", + "language_model.model.layers.34.self_attn.attn.impl.v_cache", + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.34.feed_forward.gate_up_proj", + "language_model.model.layers.34.feed_forward.down_proj", + "language_model.model.layers.35.self_attn.qkv_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.35.self_attn.attn.impl.softmax", + "language_model.model.layers.35.self_attn.attn.impl.matmul_av", + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.35.self_attn.attn.impl.k_cache", + "language_model.model.layers.35.self_attn.attn.impl.v_cache", + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.35.feed_forward.router", + "language_model.model.layers.35.feed_forward.experts", + "language_model.model.layers.35.feed_forward.experts.moe_op", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.35.feed_forward.shared_expert.down_proj", + "language_model.model.layers.36.self_attn.qkv_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.36.self_attn.attn.impl.softmax", + "language_model.model.layers.36.self_attn.attn.impl.matmul_av", + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.36.self_attn.attn.impl.k_cache", + "language_model.model.layers.36.self_attn.attn.impl.v_cache", + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.36.feed_forward.gate_up_proj", + "language_model.model.layers.36.feed_forward.down_proj", + "language_model.model.layers.37.self_attn.qkv_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.37.self_attn.attn.impl.softmax", + "language_model.model.layers.37.self_attn.attn.impl.matmul_av", + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.37.self_attn.attn.impl.k_cache", + "language_model.model.layers.37.self_attn.attn.impl.v_cache", + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.37.feed_forward.router", + "language_model.model.layers.37.feed_forward.experts", + "language_model.model.layers.37.feed_forward.experts.moe_op", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.37.feed_forward.shared_expert.down_proj", + "language_model.model.layers.38.self_attn.qkv_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.38.self_attn.attn.impl.softmax", + "language_model.model.layers.38.self_attn.attn.impl.matmul_av", + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.38.self_attn.attn.impl.k_cache", + "language_model.model.layers.38.self_attn.attn.impl.v_cache", + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.38.feed_forward.gate_up_proj", + "language_model.model.layers.38.feed_forward.down_proj", + "language_model.model.layers.39.self_attn.qkv_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.39.self_attn.attn.impl.softmax", + "language_model.model.layers.39.self_attn.attn.impl.matmul_av", + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.39.self_attn.attn.impl.k_cache", + "language_model.model.layers.39.self_attn.attn.impl.v_cache", + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.39.feed_forward.router", + "language_model.model.layers.39.feed_forward.experts", + "language_model.model.layers.39.feed_forward.experts.moe_op", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.39.feed_forward.shared_expert.down_proj", + "language_model.model.layers.40.self_attn.qkv_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.40.self_attn.attn.impl.softmax", + "language_model.model.layers.40.self_attn.attn.impl.matmul_av", + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.40.self_attn.attn.impl.k_cache", + "language_model.model.layers.40.self_attn.attn.impl.v_cache", + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.40.feed_forward.gate_up_proj", + "language_model.model.layers.40.feed_forward.down_proj", + "language_model.model.layers.41.self_attn.qkv_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.41.self_attn.attn.impl.softmax", + "language_model.model.layers.41.self_attn.attn.impl.matmul_av", + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.41.self_attn.attn.impl.k_cache", + "language_model.model.layers.41.self_attn.attn.impl.v_cache", + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.41.feed_forward.router", + "language_model.model.layers.41.feed_forward.experts", + "language_model.model.layers.41.feed_forward.experts.moe_op", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.41.feed_forward.shared_expert.down_proj", + "language_model.model.layers.42.self_attn.qkv_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.42.self_attn.attn.impl.softmax", + "language_model.model.layers.42.self_attn.attn.impl.matmul_av", + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.42.self_attn.attn.impl.k_cache", + "language_model.model.layers.42.self_attn.attn.impl.v_cache", + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.42.feed_forward.gate_up_proj", + "language_model.model.layers.42.feed_forward.down_proj", + "language_model.model.layers.43.self_attn.qkv_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.43.self_attn.attn.impl.softmax", + "language_model.model.layers.43.self_attn.attn.impl.matmul_av", + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.43.self_attn.attn.impl.k_cache", + "language_model.model.layers.43.self_attn.attn.impl.v_cache", + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.43.feed_forward.router", + "language_model.model.layers.43.feed_forward.experts", + "language_model.model.layers.43.feed_forward.experts.moe_op", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.43.feed_forward.shared_expert.down_proj", + "language_model.model.layers.44.self_attn.qkv_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.44.self_attn.attn.impl.softmax", + "language_model.model.layers.44.self_attn.attn.impl.matmul_av", + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.44.self_attn.attn.impl.k_cache", + "language_model.model.layers.44.self_attn.attn.impl.v_cache", + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.44.feed_forward.gate_up_proj", + "language_model.model.layers.44.feed_forward.down_proj", + "language_model.model.layers.45.self_attn.qkv_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.45.self_attn.attn.impl.softmax", + "language_model.model.layers.45.self_attn.attn.impl.matmul_av", + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.45.self_attn.attn.impl.k_cache", + "language_model.model.layers.45.self_attn.attn.impl.v_cache", + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.45.feed_forward.router", + "language_model.model.layers.45.feed_forward.experts", + "language_model.model.layers.45.feed_forward.experts.moe_op", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.45.feed_forward.shared_expert.down_proj", + "language_model.model.layers.46.self_attn.qkv_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.46.self_attn.attn.impl.softmax", + "language_model.model.layers.46.self_attn.attn.impl.matmul_av", + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.46.self_attn.attn.impl.k_cache", + "language_model.model.layers.46.self_attn.attn.impl.v_cache", + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.46.feed_forward.gate_up_proj", + "language_model.model.layers.46.feed_forward.down_proj", + "language_model.model.layers.47.self_attn.qkv_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.47.self_attn.attn.impl.softmax", + "language_model.model.layers.47.self_attn.attn.impl.matmul_av", + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.47.self_attn.attn.impl.k_cache", + "language_model.model.layers.47.self_attn.attn.impl.v_cache", + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.47.feed_forward.router", + "language_model.model.layers.47.feed_forward.experts", + "language_model.model.layers.47.feed_forward.experts.moe_op", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.47.feed_forward.shared_expert.down_proj", + "language_model.lm_head" +] \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_2_8.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_2_8.json new file mode 100644 index 000000000000..7d1839db3d42 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_2_8.json @@ -0,0 +1 @@ +{"GlobalRank": null, "LocalRank": 2, "Mode": "DynamicRange", "Nodes": {"language_model.model.layers.0.self_attn.qkv_proj": {"inputs": [[[11.1875]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.0.self_attn.o_proj": {"inputs": [[[0.5]]], "outputs": [[[0.80078125]], [[3.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.0.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.55078125]], [[5.625]]]}, "language_model.model.layers.0.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.50390625]]]}, "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.55078125]]]}, "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.43359375]]]}, "language_model.model.layers.0.self_attn.attn.impl.k_cache": {"inputs": [[[5.625]]]}, "language_model.model.layers.0.self_attn.attn.impl.v_cache": {"inputs": [[[0.50390625]]]}, "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[6.28125]], [[5.625]], [[0.50390625]]], "outputs": [[[0.5]], [[1.0]]]}, "language_model.model.layers.0.feed_forward.gate_up_proj": {"inputs": [[[2.546875]]], "params": {"weight": [[0.40234375]]}}, "language_model.model.layers.0.feed_forward.down_proj": {"inputs": [[[3.5]]], "outputs": [[[26.75]], [[65.0]]], "params": {"weight": [[0.75390625]]}}, "language_model.model.layers.1.self_attn.qkv_proj": {"inputs": [[[12.3125]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.1.self_attn.o_proj": {"inputs": [[[0.953125]]], "outputs": [[[0.52734375]], [[1.0078125]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.1.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.734375]], [[6.96875]]]}, "language_model.model.layers.1.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.265625]]]}, "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.734375]]]}, "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.109375]]]}, "language_model.model.layers.1.self_attn.attn.impl.k_cache": {"inputs": [[[6.96875]]]}, "language_model.model.layers.1.self_attn.attn.impl.v_cache": {"inputs": [[[1.265625]]]}, "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.875]], [[6.75]], [[1.265625]]], "outputs": [[[0.953125]], [[1.0]]]}, "language_model.model.layers.1.feed_forward.router": {"inputs": [[[1.8828125]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op": {"inputs": [[[1.8828125]]], "outputs": [[[1.0546875]], [[8.125]], [[0.0]], [[0.490234375]], [[9.1875]], [[13.0]], [[8.375]], [[11.5]], [[12.625]], [[6.625]], [[13.3125]], [[11.25]], [[11.875]], [[4.25]], [[10.5625]], [[13.375]], [[15.125]], [[3.53125]], [[3.8125]], [[14.8125]], [[12.25]], [[14.1875]], [[11.875]], [[0.0]], [[11.6875]], [[6.1875]], [[2.046875]], [[11.6875]], [[10.5625]], [[4.75]], [[9.9375]], [[17.75]], [[17.75]], [[0.166015625]], [[5.34375]], [[0.0]], [[19.125]], [[3.671875]], [[15.75]], [[8.125]], [[11.625]], [[9.0]], [[15.5625]], [[12.9375]], [[13.375]], [[4.0625]], [[13.25]], [[10.4375]], [[4.4375]], [[2.015625]], [[0.0]], [[10.0]], [[12.375]], [[8.6875]], [[2.359375]], [[7.9375]], [[14.625]], [[9.25]], [[0.89453125]], [[9.4375]], [[9.5]], [[12.1875]], [[13.25]], [[14.5]], [[0.0]], [[11.875]], [[6.28125]], [[2.171875]], [[10.0625]], [[3.796875]], [[10.9375]], [[10.1875]], [[9.9375]], [[10.1875]], [[14.5]], [[11.1875]], [[4.625]], [[10.0625]], [[10.9375]], [[4.46875]], [[13.375]], [[0.0]], [[1.734375]], [[14.75]], [[3.296875]], [[11.75]], [[11.9375]], [[11.4375]], [[12.375]], [[11.0]], [[11.625]], [[9.625]], [[11.875]], [[12.6875]], [[0.0]], [[16.625]], [[13.375]], [[0.11962890625]], [[0.0]], [[11.875]], [[19.5]], [[9.5625]], [[6.53125]], [[10.125]], [[2.765625]], [[3.3125]], [[7.375]], [[0.024169921875]], [[0.5234375]], [[3.15625]], [[0.158203125]], [[8.75]], [[0.62890625]], [[9.1875]], [[0.0]], [[14.4375]], [[16.375]], [[13.5]], [[7.40625]], [[14.0]], [[8.9375]], [[11.6875]], [[10.1875]], [[12.625]], [[2.796875]], [[9.9375]], [[10.1875]], [[12.125]], [[14.25]]]}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.4765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.07275390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.07080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.361328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[1.8828125]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.1.feed_forward.shared_expert.down_proj": {"inputs": [[[5.5]]], "outputs": [[[0.85546875]], [[1.914088313930279e+38]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.2.self_attn.qkv_proj": {"inputs": [[[9.875]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.2.self_attn.o_proj": {"inputs": [[[0.734375]]], "outputs": [[[0.6875]], [[1.328125]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.2.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.65625]], [[14.5]]]}, "language_model.model.layers.2.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.4140625]]]}, "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.65625]]]}, "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.2265625]]]}, "language_model.model.layers.2.self_attn.attn.impl.k_cache": {"inputs": [[[14.5]]]}, "language_model.model.layers.2.self_attn.attn.impl.v_cache": {"inputs": [[[1.4140625]]]}, "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.8125]], [[14.5]], [[1.4140625]]], "outputs": [[[0.734375]], [[1.0]]]}, "language_model.model.layers.2.feed_forward.gate_up_proj": {"inputs": [[[6.125]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.2.feed_forward.down_proj": {"inputs": [[[5.09375]]], "outputs": [[[4.75]], [[25.25]]], "params": {"weight": [[0.51953125]]}}, "language_model.model.layers.3.self_attn.qkv_proj": {"inputs": [[[7.21875]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.3.self_attn.o_proj": {"inputs": [[[0.458984375]]], "outputs": [[[0.59765625]], [[2.40625]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.3.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.1083984375]], [[3.0625]]]}, "language_model.model.layers.3.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.423828125]]]}, "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.1083984375]]]}, "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.353515625]]]}, "language_model.model.layers.3.self_attn.attn.impl.k_cache": {"inputs": [[[3.0625]]]}, "language_model.model.layers.3.self_attn.attn.impl.v_cache": {"inputs": [[[0.423828125]]]}, "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[1.9296875]], [[3.0625]], [[0.46484375]]], "outputs": [[[0.458984375]], [[1.0]]]}, "language_model.model.layers.3.feed_forward.router": {"inputs": [[[8.75]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op": {"inputs": [[[8.75]]], "outputs": [[[156.0]], [[0.58984375]], [[14.8125]], [[17.5]], [[4.875]], [[0.0]], [[5.96875]], [[13.6875]], [[15.0]], [[14.5625]], [[12.0625]], [[2.921875]], [[11.8125]], [[14.875]], [[11.625]], [[9.4375]], [[13.875]], [[1.046875]], [[3.890625]], [[7.9375]], [[17.25]], [[6.21875]], [[9.875]], [[12.9375]], [[14.625]], [[9.6875]], [[6.8125]], [[12.5]], [[4.46875]], [[11.125]], [[9.6875]], [[11.1875]], [[5.59375]], [[9.6875]], [[3.296875]], [[14.25]], [[11.4375]], [[5.5625]], [[6.15625]], [[15.75]], [[8.6875]], [[3.53125]], [[15.25]], [[2.03125]], [[3.140625]], [[18.875]], [[7.3125]], [[15.25]], [[12.0]], [[6.75]], [[15.1875]], [[15.8125]], [[11.375]], [[16.25]], [[3.3125]], [[11.5625]], [[8.0625]], [[3.140625]], [[9.5625]], [[15.5625]], [[12.3125]], [[6.9375]], [[15.8125]], [[13.875]], [[0.2177734375]], [[7.21875]], [[11.4375]], [[14.125]], [[0.21875]], [[6.9375]], [[0.0]], [[12.1875]], [[17.0]], [[18.375]], [[2.703125]], [[3.15625]], [[8.9375]], [[18.875]], [[0.0927734375]], [[16.875]], [[10.3125]], [[18.0]], [[0.0]], [[1.578125]], [[20.875]], [[8.6875]], [[35.75]], [[10.3125]], [[2.203125]], [[15.25]], [[14.125]], [[13.3125]], [[3.578125]], [[6.59375]], [[18.875]], [[0.0]], [[12.5625]], [[15.75]], [[17.125]], [[9.9375]], [[1.5859375]], [[12.6875]], [[5.71875]], [[3.796875]], [[16.375]], [[0.0]], [[4.1875]], [[13.875]], [[18.0]], [[10.0625]], [[3.578125]], [[11.75]], [[2.625]], [[13.625]], [[1.3125]], [[6.8125]], [[2.09375]], [[6.6875]], [[0.95703125]], [[2.390625]], [[3.09375]], [[22.875]], [[7.21875]], [[5.21875]], [[13.4375]], [[2.109375]], [[13.9375]], [[11.9375]], [[13.4375]]]}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.38671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[8.75]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.3.feed_forward.shared_expert.down_proj": {"inputs": [[[7.0]]], "outputs": [[[0.400390625]], [[1.914088313930279e+38]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.4.self_attn.qkv_proj": {"inputs": [[[13.75]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.4.self_attn.o_proj": {"inputs": [[[0.84765625]]], "outputs": [[[1.015625]], [[2.046875]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.4.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.90234375]], [[14.0625]]]}, "language_model.model.layers.4.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.140625]]]}, "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.90234375]]]}, "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.359375]]]}, "language_model.model.layers.4.self_attn.attn.impl.k_cache": {"inputs": [[[14.0625]]]}, "language_model.model.layers.4.self_attn.attn.impl.v_cache": {"inputs": [[[2.140625]]]}, "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.0]], [[14.0625]], [[2.140625]]], "outputs": [[[0.84765625]], [[1.0]]]}, "language_model.model.layers.4.feed_forward.gate_up_proj": {"inputs": [[[7.09375]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.4.feed_forward.down_proj": {"inputs": [[[6.8125]]], "outputs": [[[2.796875]], [[4.96875]]], "params": {"weight": [[0.5859375]]}}, "language_model.model.layers.5.self_attn.qkv_proj": {"inputs": [[[17.375]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.5.self_attn.o_proj": {"inputs": [[[1.296875]]], "outputs": [[[0.97265625]], [[2.21875]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.5.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.69140625]], [[12.3125]]]}, "language_model.model.layers.5.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.40625]]]}, "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.69140625]]]}, "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.484375]]]}, "language_model.model.layers.5.self_attn.attn.impl.k_cache": {"inputs": [[[12.3125]]]}, "language_model.model.layers.5.self_attn.attn.impl.v_cache": {"inputs": [[[2.40625]]]}, "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.1875]], [[12.3125]], [[2.40625]]], "outputs": [[[1.2890625]], [[1.0]]]}, "language_model.model.layers.5.feed_forward.router": {"inputs": [[[19.25]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op": {"inputs": [[[19.25]]], "outputs": [[[115.0]], [[11.0]], [[3.765625]], [[21.75]], [[10.1875]], [[11.0]], [[15.1875]], [[13.8125]], [[21.5]], [[17.125]], [[16.875]], [[7.1875]], [[17.125]], [[17.375]], [[9.125]], [[8.8125]], [[20.25]], [[10.8125]], [[13.1875]], [[14.9375]], [[29.625]], [[19.25]], [[9.5]], [[6.3125]], [[15.9375]], [[3.484375]], [[7.21875]], [[5.40625]], [[144.0]], [[11.375]], [[21.375]], [[21.0]], [[6.71875]], [[11.0]], [[21.625]], [[6.25]], [[17.375]], [[23.625]], [[10.6875]], [[12.6875]], [[13.25]], [[6.6875]], [[4.375]], [[10.6875]], [[11.25]], [[19.125]], [[8.8125]], [[9.3125]], [[15.875]], [[9.9375]], [[15.0625]], [[17.25]], [[13.6875]], [[1.3828125]], [[2.78125]], [[14.5625]], [[4.5625]], [[13.75]], [[18.5]], [[17.5]], [[4.9375]], [[7.53125]], [[14.375]], [[5.4375]], [[21.625]], [[10.375]], [[8.8125]], [[8.125]], [[9.125]], [[22.75]], [[8.8125]], [[4.125]], [[17.875]], [[16.875]], [[14.1875]], [[7.5]], [[20.875]], [[14.375]], [[11.3125]], [[21.75]], [[17.5]], [[2.015625]], [[8.5]], [[7.65625]], [[12.0625]], [[5.6875]], [[9.6875]], [[0.3046875]], [[17.5]], [[22.0]], [[21.25]], [[21.875]], [[10.75]], [[18.375]], [[20.375]], [[14.875]], [[5.75]], [[16.75]], [[9.3125]], [[4.75]], [[13.875]], [[9.125]], [[2.359375]], [[1.5390625]], [[19.5]], [[21.75]], [[8.375]], [[12.25]], [[13.75]], [[7.625]], [[13.0]], [[11.5]], [[3.453125]], [[3.5]], [[2.890625]], [[26.875]], [[11.3125]], [[2.953125]], [[5.3125]], [[1.9609375]], [[11.75]], [[14.5]], [[9.1875]], [[8.9375]], [[26.125]], [[18.75]], [[2.09375]], [[4.96875]], [[3.765625]]]}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.40234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0751953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.78125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.59375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[19.25]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.5.feed_forward.shared_expert.down_proj": {"inputs": [[[5.625]]], "outputs": [[[0.89453125]], [[1.914088313930279e+38]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.6.self_attn.qkv_proj": {"inputs": [[[12.9375]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.6.self_attn.o_proj": {"inputs": [[[1.8671875]]], "outputs": [[[0.66796875]], [[1.5234375]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.6.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.486328125]], [[7.65625]]]}, "language_model.model.layers.6.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.078125]]]}, "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.486328125]]]}, "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.171875]]]}, "language_model.model.layers.6.self_attn.attn.impl.k_cache": {"inputs": [[[7.65625]]]}, "language_model.model.layers.6.self_attn.attn.impl.v_cache": {"inputs": [[[3.078125]]]}, "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[6.90625]], [[7.65625]], [[3.078125]]], "outputs": [[[1.8671875]], [[1.0]]]}, "language_model.model.layers.6.feed_forward.gate_up_proj": {"inputs": [[[7.71875]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.6.feed_forward.down_proj": {"inputs": [[[26.375]]], "outputs": [[[2.328125]], [[11.1875]]], "params": {"weight": [[0.546875]]}}, "language_model.model.layers.7.self_attn.qkv_proj": {"inputs": [[[7.09375]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.7.self_attn.o_proj": {"inputs": [[[1.0234375]]], "outputs": [[[1.84375]], [[3.84375]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.7.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.412109375]], [[4.25]]]}, "language_model.model.layers.7.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.515625]]]}, "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.412109375]]]}, "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.79296875]]]}, "language_model.model.layers.7.self_attn.attn.impl.k_cache": {"inputs": [[[4.25]]]}, "language_model.model.layers.7.self_attn.attn.impl.v_cache": {"inputs": [[[1.546875]]]}, "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[5.40625]], [[4.25]], [[1.546875]]], "outputs": [[[1.0234375]], [[1.0]]]}, "language_model.model.layers.7.feed_forward.router": {"inputs": [[[11.6875]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op": {"inputs": [[[11.6875]]], "outputs": [[[3.46875]], [[4.03125]], [[14.125]], [[17.375]], [[15.875]], [[8.8125]], [[15.5]], [[23.25]], [[10.0]], [[9.25]], [[12.1875]], [[16.375]], [[9.75]], [[40.25]], [[22.25]], [[8.6875]], [[5.375]], [[13.5]], [[11.3125]], [[32.75]], [[16.25]], [[16.25]], [[29.375]], [[15.3125]], [[31.75]], [[24.375]], [[9.6875]], [[4.8125]], [[8.0]], [[17.625]], [[5.4375]], [[9.625]], [[13.5625]], [[7.71875]], [[4.71875]], [[7.8125]], [[10.125]], [[11.125]], [[15.9375]], [[25.0]], [[1.7421875]], [[1.1953125]], [[5.375]], [[1.828125]], [[14.5]], [[13.25]], [[16.5]], [[0.8203125]], [[18.5]], [[35.25]], [[16.75]], [[12.3125]], [[25.875]], [[16.125]], [[11.5625]], [[31.375]], [[21.625]], [[19.125]], [[2.546875]], [[4.1875]], [[6.75]], [[9.5]], [[15.5]], [[10.4375]], [[8.9375]], [[22.125]], [[13.3125]], [[9.625]], [[19.5]], [[5.6875]], [[2.296875]], [[20.75]], [[7.84375]], [[15.0]], [[15.3125]], [[26.875]], [[3.84375]], [[7.3125]], [[6.3125]], [[0.60546875]], [[17.75]], [[20.875]], [[17.125]], [[2.140625]], [[8.3125]], [[5.0625]], [[47.0]], [[1.875]], [[14.75]], [[5.4375]], [[27.0]], [[6.28125]], [[14.9375]], [[12.0]], [[5.46875]], [[20.375]], [[10.1875]], [[27.25]], [[5.625]], [[35.25]], [[7.5625]], [[13.3125]], [[19.375]], [[3.34375]], [[0.97265625]], [[19.25]], [[28.5]], [[16.875]], [[5.8125]], [[1.5078125]], [[17.125]], [[15.3125]], [[16.875]], [[6.5625]], [[55.75]], [[24.5]], [[1.9375]], [[7.03125]], [[12.8125]], [[5.53125]], [[15.375]], [[33.25]], [[45.5]], [[30.375]], [[5.21875]], [[3.421875]], [[1.5390625]], [[30.25]], [[12.5625]]]}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[11.6875]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.7.feed_forward.shared_expert.down_proj": {"inputs": [[[8.75]]], "outputs": [[[0.8984375]], [[1.914088313930279e+38]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.8.self_attn.qkv_proj": {"inputs": [[[15.125]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.8.self_attn.o_proj": {"inputs": [[[1.96875]]], "outputs": [[[1.9375]], [[2.109375]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.8.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.67578125]], [[8.75]]]}, "language_model.model.layers.8.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.75]]]}, "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.67578125]]]}, "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.6875]]]}, "language_model.model.layers.8.self_attn.attn.impl.k_cache": {"inputs": [[[8.75]]]}, "language_model.model.layers.8.self_attn.attn.impl.v_cache": {"inputs": [[[3.75]]]}, "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.875]], [[8.75]], [[3.75]]], "outputs": [[[1.96875]], [[1.0]]]}, "language_model.model.layers.8.feed_forward.gate_up_proj": {"inputs": [[[8.25]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.8.feed_forward.down_proj": {"inputs": [[[10.25]]], "outputs": [[[2.5625]], [[9.25]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.9.self_attn.qkv_proj": {"inputs": [[[15.75]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.9.self_attn.o_proj": {"inputs": [[[1.3359375]]], "outputs": [[[1.3125]], [[2.875]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.9.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.828125]], [[14.0]]]}, "language_model.model.layers.9.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.5]]]}, "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.828125]]]}, "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.8515625]]]}, "language_model.model.layers.9.self_attn.attn.impl.k_cache": {"inputs": [[[14.0]]]}, "language_model.model.layers.9.self_attn.attn.impl.v_cache": {"inputs": [[[2.5]]]}, "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.25]], [[14.0]], [[2.5]]], "outputs": [[[1.3359375]], [[1.0]]]}, "language_model.model.layers.9.feed_forward.router": {"inputs": [[[11.75]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op": {"inputs": [[[11.75]]], "outputs": [[[3.375]], [[12.75]], [[14.8125]], [[5.6875]], [[34.75]], [[20.5]], [[32.25]], [[22.75]], [[35.0]], [[25.375]], [[29.25]], [[21.0]], [[11.125]], [[38.0]], [[40.75]], [[17.375]], [[4.09375]], [[9.125]], [[25.125]], [[28.75]], [[18.5]], [[13.5625]], [[15.0625]], [[5.75]], [[23.0]], [[12.75]], [[20.125]], [[22.875]], [[24.5]], [[13.9375]], [[38.0]], [[12.5625]], [[11.3125]], [[32.75]], [[28.0]], [[42.25]], [[5.65625]], [[36.25]], [[17.375]], [[5.125]], [[21.0]], [[9.625]], [[11.9375]], [[31.375]], [[32.5]], [[14.375]], [[62.75]], [[18.25]], [[19.0]], [[21.625]], [[56.0]], [[15.4375]], [[26.875]], [[14.9375]], [[10.8125]], [[16.125]], [[7.8125]], [[41.0]], [[33.5]], [[39.75]], [[31.75]], [[12.1875]], [[13.9375]], [[15.0625]], [[50.0]], [[18.25]], [[25.0]], [[42.75]], [[16.25]], [[2.609375]], [[15.125]], [[19.625]], [[29.875]], [[9.75]], [[14.4375]], [[11.5625]], [[31.125]], [[13.1875]], [[20.25]], [[5.40625]], [[16.75]], [[17.875]], [[20.625]], [[48.75]], [[22.875]], [[20.125]], [[44.0]], [[13.6875]], [[26.875]], [[15.5625]], [[50.25]], [[3.328125]], [[8.875]], [[11.125]], [[13.4375]], [[22.75]], [[24.25]], [[11.5625]], [[13.25]], [[10.375]], [[10.0]], [[37.5]], [[14.875]], [[52.5]], [[47.5]], [[34.0]], [[16.375]], [[38.25]], [[27.75]], [[10.625]], [[29.75]], [[23.75]], [[2.640625]], [[27.0]], [[4.65625]], [[22.0]], [[42.25]], [[33.5]], [[4.46875]], [[14.8125]], [[10.75]], [[19.875]], [[10.0]], [[33.25]], [[14.125]], [[46.25]], [[18.125]], [[15.25]], [[20.5]]]}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[11.75]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.9.feed_forward.shared_expert.down_proj": {"inputs": [[[10.1875]]], "outputs": [[[1.59375]], [[1.914088313930279e+38]]], "params": {"weight": [[0.357421875]]}}, "language_model.model.layers.10.self_attn.qkv_proj": {"inputs": [[[16.125]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.10.self_attn.o_proj": {"inputs": [[[2.09375]]], "outputs": [[[1.6015625]], [[3.09375]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.10.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8125]], [[15.75]]]}, "language_model.model.layers.10.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.5]]]}, "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8125]]]}, "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.609375]]]}, "language_model.model.layers.10.self_attn.attn.impl.k_cache": {"inputs": [[[15.75]]]}, "language_model.model.layers.10.self_attn.attn.impl.v_cache": {"inputs": [[[2.5]]]}, "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.3125]], [[15.75]], [[2.5]]], "outputs": [[[2.09375]], [[1.0]]]}, "language_model.model.layers.10.feed_forward.gate_up_proj": {"inputs": [[[9.875]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.10.feed_forward.down_proj": {"inputs": [[[10.0]]], "outputs": [[[6.90625]], [[11.3125]]], "params": {"weight": [[0.57421875]]}}, "language_model.model.layers.11.self_attn.qkv_proj": {"inputs": [[[13.5625]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.11.self_attn.o_proj": {"inputs": [[[2.671875]]], "outputs": [[[8.0625]], [[7.875]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.11.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.7734375]], [[10.375]]]}, "language_model.model.layers.11.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.5625]]]}, "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.7734375]]]}, "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.140625]]]}, "language_model.model.layers.11.self_attn.attn.impl.k_cache": {"inputs": [[[10.375]]]}, "language_model.model.layers.11.self_attn.attn.impl.v_cache": {"inputs": [[[4.5625]]]}, "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.375]], [[10.375]], [[4.5625]]], "outputs": [[[2.671875]], [[1.0]]]}, "language_model.model.layers.11.feed_forward.router": {"inputs": [[[13.375]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op": {"inputs": [[[13.375]]], "outputs": [[[5.4375]], [[21.0]], [[18.875]], [[19.625]], [[12.5625]], [[31.0]], [[32.75]], [[1.4453125]], [[25.125]], [[13.375]], [[8.8125]], [[22.0]], [[18.0]], [[61.25]], [[41.25]], [[16.875]], [[46.75]], [[38.75]], [[25.5]], [[46.5]], [[16.125]], [[16.125]], [[42.75]], [[27.875]], [[14.875]], [[30.875]], [[36.5]], [[39.5]], [[8.0625]], [[9.125]], [[36.5]], [[41.75]], [[6.28125]], [[5.4375]], [[10.375]], [[37.5]], [[25.25]], [[12.9375]], [[22.875]], [[20.0]], [[24.25]], [[5.5]], [[17.5]], [[5.71875]], [[45.25]], [[11.375]], [[1.640625]], [[15.5]], [[41.0]], [[31.25]], [[10.3125]], [[54.5]], [[15.75]], [[3.421875]], [[19.0]], [[63.25]], [[32.25]], [[19.375]], [[21.375]], [[41.75]], [[23.125]], [[24.25]], [[15.5]], [[15.125]], [[18.75]], [[13.1875]], [[24.25]], [[46.5]], [[37.0]], [[15.875]], [[26.375]], [[5.65625]], [[11.9375]], [[2.4375]], [[37.25]], [[22.0]], [[14.0]], [[24.75]], [[7.46875]], [[36.0]], [[22.375]], [[18.875]], [[14.5]], [[22.75]], [[9.6875]], [[15.125]], [[18.75]], [[42.75]], [[13.125]], [[16.75]], [[24.5]], [[41.0]], [[6.1875]], [[27.0]], [[37.75]], [[26.75]], [[14.5625]], [[20.625]], [[25.25]], [[11.9375]], [[35.75]], [[10.9375]], [[53.25]], [[40.5]], [[16.25]], [[34.0]], [[22.25]], [[8.8125]], [[6.75]], [[9.625]], [[9.8125]], [[29.625]], [[5.65625]], [[2.640625]], [[13.0625]], [[32.75]], [[62.0]], [[26.25]], [[37.0]], [[45.75]], [[33.75]], [[46.5]], [[13.25]], [[11.5]], [[9.625]], [[7.6875]], [[11.125]], [[40.0]], [[22.75]]]}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.4375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[13.375]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.11.feed_forward.shared_expert.down_proj": {"inputs": [[[12.0]]], "outputs": [[[5.28125]], [[1.914088313930279e+38]]], "params": {"weight": [[0.56640625]]}}, "language_model.model.layers.12.self_attn.qkv_proj": {"inputs": [[[18.625]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.12.self_attn.o_proj": {"inputs": [[[2.21875]]], "outputs": [[[1.3359375]], [[3.640625]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.12.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0390625]], [[10.8125]]]}, "language_model.model.layers.12.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.84375]]]}, "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0390625]]]}, "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.0]]]}, "language_model.model.layers.12.self_attn.attn.impl.k_cache": {"inputs": [[[10.8125]]]}, "language_model.model.layers.12.self_attn.attn.impl.v_cache": {"inputs": [[[2.84375]]]}, "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.875]], [[10.8125]], [[2.84375]]], "outputs": [[[2.21875]], [[1.0]]]}, "language_model.model.layers.12.feed_forward.gate_up_proj": {"inputs": [[[10.75]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.12.feed_forward.down_proj": {"inputs": [[[18.75]]], "outputs": [[[9.0]], [[17.375]]], "params": {"weight": [[0.37109375]]}}, "language_model.model.layers.13.self_attn.qkv_proj": {"inputs": [[[20.875]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.13.self_attn.o_proj": {"inputs": [[[3.109375]]], "outputs": [[[1.6875]], [[6.9375]]], "params": {"weight": [[0.447265625]]}}, "language_model.model.layers.13.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.83984375]], [[15.3125]]]}, "language_model.model.layers.13.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.359375]]]}, "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.83984375]]]}, "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.828125]]]}, "language_model.model.layers.13.self_attn.attn.impl.k_cache": {"inputs": [[[15.3125]]]}, "language_model.model.layers.13.self_attn.attn.impl.v_cache": {"inputs": [[[3.359375]]]}, "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.625]], [[15.3125]], [[3.359375]]], "outputs": [[[3.109375]], [[1.0]]]}, "language_model.model.layers.13.feed_forward.router": {"inputs": [[[17.625]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op": {"inputs": [[[17.625]]], "outputs": [[[4.59375]], [[22.25]], [[41.75]], [[18.125]], [[42.5]], [[48.5]], [[12.5625]], [[25.0]], [[14.5]], [[13.125]], [[2.375]], [[37.25]], [[36.75]], [[14.125]], [[39.75]], [[16.5]], [[37.75]], [[17.625]], [[22.75]], [[22.375]], [[38.0]], [[17.25]], [[19.125]], [[8.5625]], [[4.5625]], [[12.6875]], [[36.5]], [[29.875]], [[32.75]], [[59.5]], [[8.25]], [[16.25]], [[14.875]], [[32.75]], [[49.25]], [[42.0]], [[24.875]], [[30.5]], [[42.25]], [[15.9375]], [[10.25]], [[29.875]], [[45.75]], [[25.25]], [[33.0]], [[57.5]], [[55.0]], [[25.375]], [[37.5]], [[34.0]], [[41.25]], [[24.5]], [[43.25]], [[7.40625]], [[60.75]], [[6.4375]], [[30.625]], [[37.25]], [[43.0]], [[24.875]], [[30.0]], [[5.9375]], [[14.1875]], [[11.3125]], [[31.75]], [[13.375]], [[51.5]], [[6.34375]], [[22.625]], [[42.25]], [[11.9375]], [[64.5]], [[59.75]], [[17.625]], [[13.625]], [[31.125]], [[33.5]], [[44.75]], [[35.0]], [[57.0]], [[23.375]], [[9.6875]], [[60.25]], [[3.671875]], [[38.5]], [[42.0]], [[50.5]], [[25.0]], [[7.21875]], [[7.34375]], [[50.5]], [[52.0]], [[53.75]], [[4.15625]], [[31.5]], [[48.0]], [[12.25]], [[9.625]], [[18.375]], [[22.25]], [[38.5]], [[4.5]], [[2.1875]], [[28.375]], [[28.375]], [[38.75]], [[33.5]], [[36.0]], [[8.0625]], [[21.125]], [[57.75]], [[21.125]], [[29.875]], [[10.5]], [[32.5]], [[52.25]], [[33.25]], [[10.125]], [[53.0]], [[41.5]], [[22.125]], [[28.0]], [[39.25]], [[22.125]], [[13.4375]], [[26.625]], [[22.125]], [[12.125]], [[30.375]]]}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.384765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.46484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.478515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.4921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[17.625]]], "params": {"weight": [[0.3046875]]}}, "language_model.model.layers.13.feed_forward.shared_expert.down_proj": {"inputs": [[[19.0]]], "outputs": [[[11.6875]], [[1.914088313930279e+38]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.14.self_attn.qkv_proj": {"inputs": [[[18.75]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.14.self_attn.o_proj": {"inputs": [[[2.46875]]], "outputs": [[[2.140625]], [[4.875]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.14.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.68359375]], [[11.5625]]]}, "language_model.model.layers.14.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.625]]]}, "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.68359375]]]}, "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.234375]]]}, "language_model.model.layers.14.self_attn.attn.impl.k_cache": {"inputs": [[[11.5625]]]}, "language_model.model.layers.14.self_attn.attn.impl.v_cache": {"inputs": [[[3.625]]]}, "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.3125]], [[11.5625]], [[3.625]]], "outputs": [[[2.46875]], [[1.0]]]}, "language_model.model.layers.14.feed_forward.gate_up_proj": {"inputs": [[[12.0625]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.14.feed_forward.down_proj": {"inputs": [[[16.25]]], "outputs": [[[4.65625]], [[28.25]]], "params": {"weight": [[0.78515625]]}}, "language_model.model.layers.15.self_attn.qkv_proj": {"inputs": [[[20.875]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.15.self_attn.o_proj": {"inputs": [[[3.3125]]], "outputs": [[[21.875]], [[23.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.15.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.796875]], [[10.5]]]}, "language_model.model.layers.15.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.46875]]]}, "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.796875]]]}, "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.3125]]]}, "language_model.model.layers.15.self_attn.attn.impl.k_cache": {"inputs": [[[10.5]]]}, "language_model.model.layers.15.self_attn.attn.impl.v_cache": {"inputs": [[[4.46875]]]}, "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.6875]], [[10.5]], [[4.46875]]], "outputs": [[[3.3125]], [[1.0]]]}, "language_model.model.layers.15.feed_forward.router": {"inputs": [[[19.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op": {"inputs": [[[19.0]]], "outputs": [[[4.65625]], [[9.1875]], [[18.375]], [[27.25]], [[12.25]], [[26.5]], [[37.5]], [[36.25]], [[27.0]], [[74.5]], [[46.75]], [[39.25]], [[22.0]], [[14.5625]], [[15.75]], [[3.1875]], [[18.0]], [[21.25]], [[46.5]], [[16.5]], [[8.1875]], [[32.5]], [[49.75]], [[33.0]], [[47.5]], [[9.75]], [[3.859375]], [[24.25]], [[28.25]], [[61.5]], [[13.5625]], [[29.375]], [[28.625]], [[14.25]], [[9.875]], [[22.125]], [[27.5]], [[7.21875]], [[28.875]], [[48.75]], [[32.0]], [[56.75]], [[45.5]], [[39.5]], [[26.625]], [[45.25]], [[21.625]], [[23.625]], [[29.75]], [[13.375]], [[21.75]], [[24.625]], [[42.75]], [[29.25]], [[11.0]], [[41.75]], [[42.75]], [[40.0]], [[43.25]], [[46.75]], [[42.75]], [[13.5]], [[59.75]], [[53.75]], [[53.75]], [[10.875]], [[35.0]], [[44.5]], [[65.0]], [[11.75]], [[60.25]], [[59.5]], [[10.375]], [[61.5]], [[60.75]], [[41.25]], [[34.25]], [[38.5]], [[58.0]], [[46.0]], [[26.75]], [[23.75]], [[33.0]], [[38.25]], [[9.875]], [[27.75]], [[37.75]], [[22.75]], [[16.875]], [[40.0]], [[43.75]], [[3.90625]], [[13.75]], [[24.25]], [[30.375]], [[37.75]], [[33.5]], [[30.5]], [[24.25]], [[20.125]], [[38.5]], [[30.75]], [[49.75]], [[30.25]], [[34.5]], [[43.0]], [[74.0]], [[12.8125]], [[43.5]], [[47.5]], [[17.125]], [[36.5]], [[59.5]], [[39.0]], [[9.3125]], [[17.625]], [[8.9375]], [[51.75]], [[6.75]], [[38.5]], [[20.75]], [[52.5]], [[42.0]], [[14.5625]], [[54.0]], [[66.0]], [[8.0625]], [[8.25]], [[41.5]]]}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.51171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[19.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.15.feed_forward.shared_expert.down_proj": {"inputs": [[[16.625]]], "outputs": [[[2.640625]], [[1.914088313930279e+38]]], "params": {"weight": [[0.357421875]]}}, "language_model.model.layers.16.self_attn.qkv_proj": {"inputs": [[[20.375]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.16.self_attn.o_proj": {"inputs": [[[5.1875]]], "outputs": [[[2.75]], [[6.28125]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.16.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.03125]], [[12.0625]]]}, "language_model.model.layers.16.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.90625]]]}, "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.03125]]]}, "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.6875]]]}, "language_model.model.layers.16.self_attn.attn.impl.k_cache": {"inputs": [[[12.0625]]]}, "language_model.model.layers.16.self_attn.attn.impl.v_cache": {"inputs": [[[6.90625]]]}, "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.875]], [[12.0625]], [[6.90625]]], "outputs": [[[5.1875]], [[1.0]]]}, "language_model.model.layers.16.feed_forward.gate_up_proj": {"inputs": [[[14.8125]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.16.feed_forward.down_proj": {"inputs": [[[14.8125]]], "outputs": [[[7.03125]], [[22.625]]], "params": {"weight": [[0.53125]]}}, "language_model.model.layers.17.self_attn.qkv_proj": {"inputs": [[[19.875]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.17.self_attn.o_proj": {"inputs": [[[5.21875]]], "outputs": [[[17.5]], [[20.75]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.17.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.7421875]], [[10.375]]]}, "language_model.model.layers.17.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.6875]]]}, "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.7421875]]]}, "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.875]]]}, "language_model.model.layers.17.self_attn.attn.impl.k_cache": {"inputs": [[[10.375]]]}, "language_model.model.layers.17.self_attn.attn.impl.v_cache": {"inputs": [[[5.6875]]]}, "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.125]], [[10.375]], [[5.6875]]], "outputs": [[[5.21875]], [[1.0]]]}, "language_model.model.layers.17.feed_forward.router": {"inputs": [[[23.875]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op": {"inputs": [[[23.875]]], "outputs": [[[5.78125]], [[27.125]], [[10.9375]], [[25.75]], [[19.5]], [[38.0]], [[26.0]], [[65.5]], [[27.0]], [[47.25]], [[5.75]], [[20.5]], [[22.5]], [[19.0]], [[23.25]], [[33.25]], [[41.0]], [[29.25]], [[32.25]], [[14.75]], [[63.75]], [[39.5]], [[48.75]], [[25.375]], [[41.5]], [[25.5]], [[45.75]], [[26.0]], [[32.25]], [[52.75]], [[24.125]], [[55.75]], [[17.5]], [[10.5]], [[81.0]], [[25.625]], [[75.0]], [[39.25]], [[29.375]], [[25.25]], [[27.25]], [[19.625]], [[51.75]], [[6.5]], [[20.0]], [[31.375]], [[50.5]], [[26.25]], [[56.0]], [[26.5]], [[23.625]], [[71.0]], [[51.25]], [[9.1875]], [[16.25]], [[18.125]], [[6.59375]], [[7.15625]], [[65.0]], [[23.5]], [[29.25]], [[14.5625]], [[54.75]], [[10.0625]], [[18.25]], [[34.0]], [[37.5]], [[10.375]], [[33.75]], [[33.0]], [[17.25]], [[40.75]], [[12.75]], [[67.0]], [[29.0]], [[16.875]], [[13.125]], [[41.5]], [[38.0]], [[10.9375]], [[36.75]], [[13.0]], [[38.0]], [[30.625]], [[46.75]], [[49.0]], [[44.0]], [[13.125]], [[38.5]], [[55.5]], [[48.75]], [[53.0]], [[37.75]], [[6.5625]], [[34.5]], [[17.625]], [[35.0]], [[13.0625]], [[3.546875]], [[27.0]], [[21.75]], [[10.5]], [[44.25]], [[18.375]], [[32.75]], [[40.0]], [[17.25]], [[19.0]], [[45.75]], [[15.25]], [[11.25]], [[62.5]], [[28.5]], [[51.75]], [[23.375]], [[15.3125]], [[12.875]], [[75.0]], [[25.75]], [[24.625]], [[43.5]], [[36.5]], [[18.0]], [[35.25]], [[17.5]], [[22.125]], [[7.5625]], [[8.4375]], [[59.75]]]}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.3046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.875]]], "params": {"weight": [[0.365234375]]}}, "language_model.model.layers.17.feed_forward.shared_expert.down_proj": {"inputs": [[[21.375]]], "outputs": [[[19.875]], [[1.914088313930279e+38]]], "params": {"weight": [[0.92578125]]}}, "language_model.model.layers.18.self_attn.qkv_proj": {"inputs": [[[22.125]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.18.self_attn.o_proj": {"inputs": [[[3.40625]]], "outputs": [[[6.375]], [[11.5625]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.18.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0078125]], [[16.75]]]}, "language_model.model.layers.18.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.65625]]]}, "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0078125]]]}, "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.734375]]]}, "language_model.model.layers.18.self_attn.attn.impl.k_cache": {"inputs": [[[16.75]]]}, "language_model.model.layers.18.self_attn.attn.impl.v_cache": {"inputs": [[[3.8125]]]}, "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.6875]], [[16.75]], [[3.8125]]], "outputs": [[[3.40625]], [[1.0]]]}, "language_model.model.layers.18.feed_forward.gate_up_proj": {"inputs": [[[18.5]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.18.feed_forward.down_proj": {"inputs": [[[266.0]]], "outputs": [[[8.25]], [[358.0]]], "params": {"weight": [[0.70703125]]}}, "language_model.model.layers.19.self_attn.qkv_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.19.self_attn.o_proj": {"inputs": [[[4.1875]]], "outputs": [[[31.625]], [[36.75]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.19.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.828125]], [[10.25]]]}, "language_model.model.layers.19.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.3125]]]}, "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.828125]]]}, "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.0625]]]}, "language_model.model.layers.19.self_attn.attn.impl.k_cache": {"inputs": [[[10.25]]]}, "language_model.model.layers.19.self_attn.attn.impl.v_cache": {"inputs": [[[5.3125]]]}, "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.875]], [[10.25]], [[5.3125]]], "outputs": [[[4.1875]], [[1.0]]]}, "language_model.model.layers.19.feed_forward.router": {"inputs": [[[23.25]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op": {"inputs": [[[23.25]]], "outputs": [[[4.25]], [[25.0]], [[63.75]], [[22.125]], [[59.25]], [[58.5]], [[20.75]], [[15.875]], [[50.5]], [[20.0]], [[12.875]], [[11.75]], [[34.5]], [[25.0]], [[28.125]], [[26.125]], [[10.3125]], [[10.5625]], [[47.0]], [[30.125]], [[39.25]], [[16.875]], [[30.75]], [[34.75]], [[23.5]], [[32.0]], [[47.0]], [[41.0]], [[33.5]], [[33.25]], [[48.25]], [[13.25]], [[37.75]], [[32.25]], [[25.625]], [[49.25]], [[40.75]], [[18.5]], [[29.5]], [[38.75]], [[13.25]], [[46.0]], [[16.125]], [[36.25]], [[32.25]], [[38.75]], [[16.0]], [[18.125]], [[16.125]], [[27.5]], [[72.5]], [[8.25]], [[32.0]], [[6.15625]], [[35.25]], [[32.25]], [[33.0]], [[23.25]], [[55.25]], [[50.5]], [[35.0]], [[14.6875]], [[42.5]], [[15.25]], [[13.5625]], [[64.0]], [[55.25]], [[49.75]], [[25.625]], [[34.25]], [[9.4375]], [[36.0]], [[25.75]], [[35.5]], [[23.0]], [[22.875]], [[16.375]], [[20.625]], [[25.875]], [[40.0]], [[9.625]], [[39.5]], [[26.75]], [[23.375]], [[51.25]], [[23.0]], [[28.375]], [[14.875]], [[5.78125]], [[13.5625]], [[28.625]], [[37.5]], [[43.0]], [[27.125]], [[34.75]], [[20.375]], [[37.75]], [[42.0]], [[22.5]], [[12.5625]], [[13.25]], [[59.0]], [[32.25]], [[17.625]], [[30.375]], [[37.5]], [[64.5]], [[33.0]], [[15.8125]], [[26.0]], [[48.75]], [[18.375]], [[11.375]], [[42.75]], [[42.5]], [[31.75]], [[8.8125]], [[16.75]], [[46.25]], [[31.75]], [[37.25]], [[11.3125]], [[17.5]], [[24.875]], [[23.375]], [[9.75]], [[19.375]], [[28.375]], [[44.5]]]}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.5546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.341796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.36328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.404296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.25]]], "params": {"weight": [[0.50390625]]}}, "language_model.model.layers.19.feed_forward.shared_expert.down_proj": {"inputs": [[[23.75]]], "outputs": [[[2.9375]], [[1.914088313930279e+38]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.20.self_attn.qkv_proj": {"inputs": [[[52.5]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.20.self_attn.o_proj": {"inputs": [[[4.71875]]], "outputs": [[[9.5]], [[18.375]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.20.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.2734375]], [[16.25]]]}, "language_model.model.layers.20.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.75]]]}, "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.2734375]]]}, "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.4375]]]}, "language_model.model.layers.20.self_attn.attn.impl.k_cache": {"inputs": [[[16.25]]]}, "language_model.model.layers.20.self_attn.attn.impl.v_cache": {"inputs": [[[6.75]]]}, "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[16.25]], [[16.25]], [[6.75]]], "outputs": [[[4.71875]], [[1.0]]]}, "language_model.model.layers.20.feed_forward.gate_up_proj": {"inputs": [[[41.5]]], "params": {"weight": [[0.32421875]]}}, "language_model.model.layers.20.feed_forward.down_proj": {"inputs": [[[23.125]]], "outputs": [[[9.4375]], [[38.5]]], "params": {"weight": [[0.84375]]}}, "language_model.model.layers.21.self_attn.qkv_proj": {"inputs": [[[56.25]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.21.self_attn.o_proj": {"inputs": [[[5.0625]]], "outputs": [[[25.125]], [[37.75]]], "params": {"weight": [[0.6484375]]}}, "language_model.model.layers.21.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0078125]], [[12.4375]]]}, "language_model.model.layers.21.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.59375]]]}, "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0078125]]]}, "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.84375]]]}, "language_model.model.layers.21.self_attn.attn.impl.k_cache": {"inputs": [[[12.4375]]]}, "language_model.model.layers.21.self_attn.attn.impl.v_cache": {"inputs": [[[6.59375]]]}, "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.0]], [[12.4375]], [[6.59375]]], "outputs": [[[5.0625]], [[1.0]]]}, "language_model.model.layers.21.feed_forward.router": {"inputs": [[[24.5]]], "params": {"weight": [[0.45703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op": {"inputs": [[[24.5]]], "outputs": [[[39.0]], [[21.5]], [[81.0]], [[40.75]], [[45.0]], [[9.375]], [[40.5]], [[37.25]], [[29.75]], [[24.0]], [[25.5]], [[19.625]], [[11.9375]], [[34.75]], [[68.5]], [[19.75]], [[50.25]], [[63.75]], [[48.75]], [[34.25]], [[60.25]], [[37.5]], [[29.125]], [[23.0]], [[16.375]], [[36.25]], [[13.75]], [[9.1875]], [[21.75]], [[9.4375]], [[28.75]], [[11.875]], [[44.75]], [[48.0]], [[12.8125]], [[14.625]], [[13.0]], [[28.875]], [[18.5]], [[37.25]], [[28.75]], [[7.59375]], [[28.75]], [[22.375]], [[5.875]], [[26.25]], [[20.5]], [[15.9375]], [[26.25]], [[23.75]], [[23.875]], [[39.0]], [[28.625]], [[39.0]], [[27.5]], [[10.5]], [[43.75]], [[29.75]], [[16.25]], [[10.1875]], [[38.0]], [[34.5]], [[40.0]], [[19.875]], [[51.0]], [[15.75]], [[78.5]], [[16.0]], [[29.875]], [[9.5625]], [[17.75]], [[21.125]], [[20.125]], [[31.0]], [[31.875]], [[20.875]], [[19.75]], [[54.75]], [[10.9375]], [[17.5]], [[16.0]], [[33.0]], [[19.0]], [[54.75]], [[42.0]], [[24.125]], [[43.25]], [[9.6875]], [[62.25]], [[10.0]], [[29.375]], [[19.25]], [[6.46875]], [[60.0]], [[26.375]], [[19.0]], [[20.0]], [[16.75]], [[21.5]], [[36.0]], [[26.75]], [[15.0625]], [[37.75]], [[39.25]], [[39.25]], [[25.5]], [[29.125]], [[20.375]], [[28.375]], [[35.75]], [[24.875]], [[39.75]], [[45.0]], [[17.5]], [[12.8125]], [[23.625]], [[10.25]], [[64.5]], [[13.3125]], [[16.75]], [[30.875]], [[8.6875]], [[40.5]], [[27.0]], [[24.375]], [[18.5]], [[70.5]], [[50.25]], [[49.75]]]}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.80859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[24.5]]], "params": {"weight": [[0.392578125]]}}, "language_model.model.layers.21.feed_forward.shared_expert.down_proj": {"inputs": [[[22.5]]], "outputs": [[[4.03125]], [[1.914088313930279e+38]]], "params": {"weight": [[0.50390625]]}}, "language_model.model.layers.22.self_attn.qkv_proj": {"inputs": [[[55.75]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.22.self_attn.o_proj": {"inputs": [[[5.5625]]], "outputs": [[[17.25]], [[23.25]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.22.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.984375]], [[16.5]]]}, "language_model.model.layers.22.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.71875]]]}, "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.984375]]]}, "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.625]]]}, "language_model.model.layers.22.self_attn.attn.impl.k_cache": {"inputs": [[[16.5]]]}, "language_model.model.layers.22.self_attn.attn.impl.v_cache": {"inputs": [[[7.71875]]]}, "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.0]], [[16.5]], [[7.71875]]], "outputs": [[[5.5625]], [[1.0]]]}, "language_model.model.layers.22.feed_forward.gate_up_proj": {"inputs": [[[44.75]]], "params": {"weight": [[0.50390625]]}}, "language_model.model.layers.22.feed_forward.down_proj": {"inputs": [[[15.8125]]], "outputs": [[[12.0]], [[80.5]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.23.self_attn.qkv_proj": {"inputs": [[[67.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.23.self_attn.o_proj": {"inputs": [[[4.875]]], "outputs": [[[50.5]], [[61.5]]], "params": {"weight": [[0.80859375]]}}, "language_model.model.layers.23.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8125]], [[14.8125]]]}, "language_model.model.layers.23.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.625]]]}, "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8125]]]}, "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.875]]]}, "language_model.model.layers.23.self_attn.attn.impl.k_cache": {"inputs": [[[14.8125]]]}, "language_model.model.layers.23.self_attn.attn.impl.v_cache": {"inputs": [[[8.625]]]}, "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.1875]], [[14.8125]], [[8.625]]], "outputs": [[[4.875]], [[1.0]]]}, "language_model.model.layers.23.feed_forward.router": {"inputs": [[[23.625]]], "params": {"weight": [[0.474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op": {"inputs": [[[23.625]]], "outputs": [[[6.6875]], [[59.5]], [[10.0]], [[26.25]], [[67.0]], [[7.375]], [[37.5]], [[23.375]], [[37.25]], [[74.0]], [[27.5]], [[40.25]], [[48.25]], [[33.0]], [[13.0]], [[34.25]], [[44.75]], [[29.875]], [[28.375]], [[28.75]], [[31.0]], [[55.5]], [[85.0]], [[8.25]], [[25.5]], [[64.0]], [[18.875]], [[37.5]], [[75.5]], [[16.75]], [[12.8125]], [[20.25]], [[32.0]], [[25.375]], [[17.125]], [[29.375]], [[32.75]], [[11.6875]], [[15.0625]], [[16.625]], [[56.0]], [[64.5]], [[13.3125]], [[48.5]], [[8.75]], [[17.5]], [[9.25]], [[40.75]], [[58.25]], [[9.3125]], [[60.25]], [[29.0]], [[46.0]], [[76.5]], [[20.75]], [[18.375]], [[51.25]], [[40.25]], [[8.0625]], [[52.25]], [[21.875]], [[18.5]], [[33.0]], [[12.0625]], [[8.0625]], [[10.75]], [[8.5625]], [[38.75]], [[81.5]], [[34.5]], [[82.0]], [[6.34375]], [[25.0]], [[32.0]], [[37.5]], [[14.1875]], [[34.0]], [[43.0]], [[68.5]], [[28.125]], [[25.125]], [[24.75]], [[28.0]], [[30.875]], [[67.0]], [[6.5625]], [[50.5]], [[13.875]], [[8.8125]], [[26.75]], [[15.0625]], [[64.5]], [[47.25]], [[28.25]], [[10.375]], [[16.25]], [[29.625]], [[28.375]], [[29.0]], [[31.5]], [[67.0]], [[10.9375]], [[14.125]], [[12.0625]], [[12.625]], [[23.5]], [[63.75]], [[73.0]], [[36.25]], [[9.125]], [[88.0]], [[11.375]], [[53.0]], [[9.625]], [[58.0]], [[28.375]], [[12.375]], [[19.875]], [[51.0]], [[16.375]], [[10.8125]], [[14.375]], [[23.375]], [[28.75]], [[48.75]], [[47.0]], [[25.5]], [[12.6875]], [[8.375]]]}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.48046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.41796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.51953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.478515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.625]]], "params": {"weight": [[0.34765625]]}}, "language_model.model.layers.23.feed_forward.shared_expert.down_proj": {"inputs": [[[23.375]]], "outputs": [[[6.28125]], [[1.914088313930279e+38]]], "params": {"weight": [[0.490234375]]}}, "language_model.model.layers.24.self_attn.qkv_proj": {"inputs": [[[60.5]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.24.self_attn.o_proj": {"inputs": [[[6.5625]]], "outputs": [[[36.0]], [[42.5]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.24.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.9453125]], [[16.5]]]}, "language_model.model.layers.24.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.375]]]}, "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.9453125]]]}, "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.71875]]]}, "language_model.model.layers.24.self_attn.attn.impl.k_cache": {"inputs": [[[16.5]]]}, "language_model.model.layers.24.self_attn.attn.impl.v_cache": {"inputs": [[[11.375]]]}, "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.9375]], [[16.5]], [[11.375]]], "outputs": [[[6.5625]], [[1.0]]]}, "language_model.model.layers.24.feed_forward.gate_up_proj": {"inputs": [[[40.5]]], "params": {"weight": [[0.494140625]]}}, "language_model.model.layers.24.feed_forward.down_proj": {"inputs": [[[24.625]]], "outputs": [[[13.3125]], [[62.5]]], "params": {"weight": [[0.5546875]]}}, "language_model.model.layers.25.self_attn.qkv_proj": {"inputs": [[[49.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.25.self_attn.o_proj": {"inputs": [[[6.0625]]], "outputs": [[[47.75]], [[57.0]]], "params": {"weight": [[0.39453125]]}}, "language_model.model.layers.25.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.85546875]], [[11.8125]]]}, "language_model.model.layers.25.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[14.375]]]}, "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.85546875]]]}, "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.5625]]]}, "language_model.model.layers.25.self_attn.attn.impl.k_cache": {"inputs": [[[11.8125]]]}, "language_model.model.layers.25.self_attn.attn.impl.v_cache": {"inputs": [[[14.375]]]}, "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.875]], [[11.8125]], [[14.375]]], "outputs": [[[6.0625]], [[1.0]]]}, "language_model.model.layers.25.feed_forward.router": {"inputs": [[[30.375]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op": {"inputs": [[[30.375]]], "outputs": [[[6.59375]], [[40.75]], [[42.75]], [[18.625]], [[19.25]], [[29.5]], [[47.25]], [[12.5]], [[36.75]], [[14.1875]], [[15.9375]], [[6.78125]], [[20.875]], [[37.5]], [[71.5]], [[40.25]], [[29.25]], [[16.75]], [[23.625]], [[40.25]], [[36.5]], [[29.75]], [[52.0]], [[50.0]], [[29.625]], [[9.9375]], [[10.4375]], [[66.5]], [[24.0]], [[11.0]], [[47.75]], [[26.25]], [[11.5]], [[9.5625]], [[64.0]], [[24.875]], [[9.4375]], [[20.375]], [[19.25]], [[18.875]], [[73.0]], [[35.5]], [[45.75]], [[36.0]], [[51.0]], [[43.5]], [[50.75]], [[92.0]], [[14.0625]], [[12.3125]], [[48.75]], [[24.25]], [[12.875]], [[19.875]], [[9.5625]], [[34.0]], [[21.0]], [[21.5]], [[17.875]], [[10.875]], [[12.625]], [[16.25]], [[49.25]], [[18.375]], [[13.4375]], [[102.5]], [[15.0]], [[37.25]], [[33.0]], [[10.5625]], [[16.125]], [[12.75]], [[20.125]], [[29.0]], [[23.375]], [[74.0]], [[98.5]], [[14.0625]], [[12.9375]], [[25.0]], [[55.25]], [[27.375]], [[39.75]], [[30.0]], [[33.75]], [[9.125]], [[32.5]], [[31.75]], [[34.25]], [[27.75]], [[52.25]], [[13.75]], [[10.8125]], [[38.75]], [[20.125]], [[8.625]], [[44.0]], [[35.75]], [[38.5]], [[15.5625]], [[36.0]], [[19.375]], [[33.5]], [[28.25]], [[21.375]], [[24.125]], [[20.375]], [[35.0]], [[10.125]], [[46.75]], [[22.375]], [[36.25]], [[32.0]], [[71.0]], [[51.25]], [[34.0]], [[39.0]], [[68.5]], [[16.875]], [[59.0]], [[42.5]], [[27.875]], [[35.0]], [[31.375]], [[21.375]], [[45.75]], [[7.46875]], [[15.8125]], [[17.0]]]}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.3671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[30.375]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.25.feed_forward.shared_expert.down_proj": {"inputs": [[[41.0]]], "outputs": [[[28.875]], [[1.914088313930279e+38]]], "params": {"weight": [[0.76953125]]}}, "language_model.model.layers.26.self_attn.qkv_proj": {"inputs": [[[63.25]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.26.self_attn.o_proj": {"inputs": [[[4.0625]]], "outputs": [[[29.0]], [[53.0]]], "params": {"weight": [[0.6796875]]}}, "language_model.model.layers.26.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.98828125]], [[15.125]]]}, "language_model.model.layers.26.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.21875]]]}, "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.98828125]]]}, "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.1875]]]}, "language_model.model.layers.26.self_attn.attn.impl.k_cache": {"inputs": [[[15.125]]]}, "language_model.model.layers.26.self_attn.attn.impl.v_cache": {"inputs": [[[6.21875]]]}, "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.75]], [[15.125]], [[6.21875]]], "outputs": [[[4.0625]], [[1.0]]]}, "language_model.model.layers.26.feed_forward.gate_up_proj": {"inputs": [[[45.5]]], "params": {"weight": [[0.451171875]]}}, "language_model.model.layers.26.feed_forward.down_proj": {"inputs": [[[22.125]]], "outputs": [[[15.25]], [[103.0]]], "params": {"weight": [[0.53515625]]}}, "language_model.model.layers.27.self_attn.qkv_proj": {"inputs": [[[72.5]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.self_attn.o_proj": {"inputs": [[[5.71875]]], "outputs": [[[60.0]], [[66.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.27.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.69140625]], [[14.8125]]]}, "language_model.model.layers.27.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.25]]]}, "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.69140625]]]}, "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.46875]]]}, "language_model.model.layers.27.self_attn.attn.impl.k_cache": {"inputs": [[[14.8125]]]}, "language_model.model.layers.27.self_attn.attn.impl.v_cache": {"inputs": [[[9.25]]]}, "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.25]], [[14.8125]], [[9.25]]], "outputs": [[[5.71875]], [[1.0]]]}, "language_model.model.layers.27.feed_forward.router": {"inputs": [[[30.875]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op": {"inputs": [[[30.875]]], "outputs": [[[6.875]], [[21.25]], [[37.5]], [[37.25]], [[11.5625]], [[12.875]], [[35.0]], [[27.25]], [[47.75]], [[65.5]], [[29.75]], [[36.5]], [[28.125]], [[50.25]], [[15.75]], [[58.25]], [[28.5]], [[19.375]], [[113.5]], [[29.875]], [[20.5]], [[16.375]], [[10.875]], [[11.9375]], [[18.125]], [[22.0]], [[26.375]], [[35.25]], [[15.25]], [[64.0]], [[37.0]], [[94.0]], [[12.875]], [[44.0]], [[11.5]], [[70.0]], [[12.0625]], [[38.0]], [[67.0]], [[38.0]], [[29.375]], [[19.125]], [[40.25]], [[52.5]], [[35.5]], [[11.125]], [[37.5]], [[23.375]], [[19.125]], [[44.5]], [[11.25]], [[18.875]], [[7.9375]], [[49.0]], [[61.5]], [[42.5]], [[29.75]], [[33.0]], [[14.625]], [[55.25]], [[10.0625]], [[34.25]], [[34.5]], [[41.25]], [[12.0625]], [[52.25]], [[103.0]], [[13.5625]], [[18.625]], [[14.4375]], [[9.75]], [[14.875]], [[26.25]], [[94.5]], [[26.125]], [[48.0]], [[55.75]], [[37.75]], [[57.75]], [[30.5]], [[35.25]], [[66.5]], [[40.25]], [[30.875]], [[64.0]], [[28.25]], [[42.5]], [[12.625]], [[17.875]], [[10.375]], [[21.125]], [[10.6875]], [[46.75]], [[54.25]], [[51.75]], [[44.25]], [[36.0]], [[25.25]], [[39.5]], [[14.5625]], [[33.25]], [[45.5]], [[37.75]], [[15.3125]], [[42.0]], [[14.0625]], [[31.5]], [[14.0625]], [[30.125]], [[81.0]], [[51.5]], [[15.125]], [[73.0]], [[13.75]], [[26.875]], [[23.25]], [[24.375]], [[36.5]], [[47.75]], [[30.5]], [[77.5]], [[20.375]], [[32.0]], [[44.5]], [[82.5]], [[13.4375]], [[35.0]], [[25.0]], [[28.75]]]}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[30.875]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.27.feed_forward.shared_expert.down_proj": {"inputs": [[[61.25]]], "outputs": [[[47.0]], [[1.914088313930279e+38]]], "params": {"weight": [[1.03125]]}}, "language_model.model.layers.28.self_attn.qkv_proj": {"inputs": [[[77.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.28.self_attn.o_proj": {"inputs": [[[4.5625]]], "outputs": [[[72.0]], [[74.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.28.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.90234375]], [[15.625]]]}, "language_model.model.layers.28.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.9375]]]}, "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.90234375]]]}, "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.34375]]]}, "language_model.model.layers.28.self_attn.attn.impl.k_cache": {"inputs": [[[15.625]]]}, "language_model.model.layers.28.self_attn.attn.impl.v_cache": {"inputs": [[[9.9375]]]}, "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.3125]], [[15.625]], [[9.9375]]], "outputs": [[[4.5625]], [[1.0]]]}, "language_model.model.layers.28.feed_forward.gate_up_proj": {"inputs": [[[40.25]]], "params": {"weight": [[0.62109375]]}}, "language_model.model.layers.28.feed_forward.down_proj": {"inputs": [[[33.75]]], "outputs": [[[95.0]], [[103.5]]], "params": {"weight": [[0.828125]]}}, "language_model.model.layers.29.self_attn.qkv_proj": {"inputs": [[[66.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.29.self_attn.o_proj": {"inputs": [[[7.21875]]], "outputs": [[[70.5]], [[76.0]]], "params": {"weight": [[0.91796875]]}}, "language_model.model.layers.29.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.2890625]], [[19.0]]]}, "language_model.model.layers.29.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.25]]]}, "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.2890625]]]}, "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.125]]]}, "language_model.model.layers.29.self_attn.attn.impl.k_cache": {"inputs": [[[19.5]]]}, "language_model.model.layers.29.self_attn.attn.impl.v_cache": {"inputs": [[[8.25]]]}, "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.6875]], [[19.5]], [[8.25]]], "outputs": [[[7.21875]], [[1.0]]]}, "language_model.model.layers.29.feed_forward.router": {"inputs": [[[31.75]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op": {"inputs": [[[31.75]]], "outputs": [[[7.09375]], [[11.0]], [[36.75]], [[65.5]], [[37.0]], [[25.25]], [[36.75]], [[10.125]], [[45.25]], [[32.75]], [[107.0]], [[11.9375]], [[15.6875]], [[14.25]], [[39.75]], [[35.5]], [[8.1875]], [[75.5]], [[47.25]], [[14.625]], [[12.5]], [[28.625]], [[13.875]], [[26.375]], [[11.625]], [[48.0]], [[22.75]], [[30.25]], [[18.625]], [[15.9375]], [[43.75]], [[41.25]], [[20.25]], [[35.75]], [[56.5]], [[13.375]], [[25.0]], [[39.0]], [[49.0]], [[73.0]], [[48.5]], [[12.125]], [[25.375]], [[12.4375]], [[55.5]], [[25.625]], [[47.5]], [[22.25]], [[41.25]], [[33.0]], [[17.5]], [[18.75]], [[22.0]], [[45.5]], [[43.75]], [[47.75]], [[48.0]], [[31.0]], [[22.625]], [[34.25]], [[14.6875]], [[79.0]], [[15.5]], [[23.5]], [[18.75]], [[40.5]], [[47.25]], [[31.5]], [[51.5]], [[55.75]], [[19.75]], [[18.875]], [[26.5]], [[19.375]], [[29.5]], [[22.5]], [[58.5]], [[13.375]], [[51.0]], [[55.5]], [[14.5]], [[42.25]], [[17.125]], [[27.875]], [[35.0]], [[23.125]], [[29.0]], [[18.25]], [[13.4375]], [[22.25]], [[14.75]], [[17.75]], [[31.0]], [[25.875]], [[23.25]], [[6.59375]], [[24.125]], [[20.75]], [[24.25]], [[43.75]], [[16.75]], [[27.625]], [[33.25]], [[15.1875]], [[48.75]], [[10.0]], [[118.0]], [[69.5]], [[53.75]], [[15.5625]], [[21.0]], [[25.125]], [[39.0]], [[57.25]], [[13.375]], [[50.25]], [[27.75]], [[32.75]], [[36.0]], [[80.5]], [[32.5]], [[40.75]], [[24.125]], [[61.0]], [[55.75]], [[22.75]], [[35.5]], [[106.5]], [[19.0]]]}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[31.75]]], "params": {"weight": [[0.365234375]]}}, "language_model.model.layers.29.feed_forward.shared_expert.down_proj": {"inputs": [[[36.75]]], "outputs": [[[17.125]], [[1.914088313930279e+38]]], "params": {"weight": [[0.8046875]]}}, "language_model.model.layers.30.self_attn.qkv_proj": {"inputs": [[[71.5]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.30.self_attn.o_proj": {"inputs": [[[6.9375]]], "outputs": [[[52.25]], [[57.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.30.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.4375]], [[18.875]]]}, "language_model.model.layers.30.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[10.25]]]}, "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.4375]]]}, "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.25]]]}, "language_model.model.layers.30.self_attn.attn.impl.k_cache": {"inputs": [[[18.875]]]}, "language_model.model.layers.30.self_attn.attn.impl.v_cache": {"inputs": [[[10.25]]]}, "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[16.5]], [[18.875]], [[10.25]]], "outputs": [[[6.9375]], [[1.0]]]}, "language_model.model.layers.30.feed_forward.gate_up_proj": {"inputs": [[[32.0]]], "params": {"weight": [[0.484375]]}}, "language_model.model.layers.30.feed_forward.down_proj": {"inputs": [[[31.5]]], "outputs": [[[18.625]], [[139.0]]], "params": {"weight": [[0.609375]]}}, "language_model.model.layers.31.self_attn.qkv_proj": {"inputs": [[[71.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.31.self_attn.o_proj": {"inputs": [[[7.09375]]], "outputs": [[[66.5]], [[88.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.31.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.95703125]], [[11.8125]]]}, "language_model.model.layers.31.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[16.625]]]}, "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.95703125]]]}, "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.28125]]]}, "language_model.model.layers.31.self_attn.attn.impl.k_cache": {"inputs": [[[11.8125]]]}, "language_model.model.layers.31.self_attn.attn.impl.v_cache": {"inputs": [[[16.625]]]}, "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.625]], [[11.8125]], [[16.625]]], "outputs": [[[7.09375]], [[1.0]]]}, "language_model.model.layers.31.feed_forward.router": {"inputs": [[[33.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op": {"inputs": [[[33.0]]], "outputs": [[[7.375]], [[12.0625]], [[106.0]], [[17.5]], [[41.0]], [[86.0]], [[37.25]], [[52.0]], [[13.6875]], [[22.875]], [[54.0]], [[29.0]], [[12.375]], [[12.25]], [[14.5625]], [[30.375]], [[11.9375]], [[28.0]], [[30.375]], [[88.5]], [[15.9375]], [[10.0625]], [[23.0]], [[34.25]], [[46.75]], [[85.0]], [[53.75]], [[53.25]], [[54.25]], [[16.75]], [[11.8125]], [[35.0]], [[26.625]], [[23.375]], [[74.0]], [[14.75]], [[18.625]], [[31.625]], [[38.75]], [[30.25]], [[70.0]], [[53.5]], [[49.5]], [[33.0]], [[57.5]], [[57.5]], [[62.0]], [[29.0]], [[31.5]], [[19.625]], [[82.0]], [[41.5]], [[29.5]], [[50.5]], [[16.25]], [[30.375]], [[26.5]], [[10.625]], [[75.5]], [[28.5]], [[29.75]], [[40.0]], [[36.5]], [[38.75]], [[41.75]], [[56.75]], [[53.25]], [[18.625]], [[22.75]], [[55.25]], [[24.5]], [[47.75]], [[35.25]], [[27.625]], [[69.5]], [[12.6875]], [[12.5625]], [[31.0]], [[51.5]], [[27.75]], [[25.875]], [[20.625]], [[94.5]], [[23.5]], [[78.0]], [[16.875]], [[16.25]], [[66.5]], [[18.875]], [[36.75]], [[60.75]], [[70.5]], [[25.5]], [[24.625]], [[14.4375]], [[46.75]], [[32.75]], [[65.0]], [[42.5]], [[19.875]], [[31.0]], [[27.875]], [[14.375]], [[43.75]], [[24.875]], [[55.75]], [[6.71875]], [[30.75]], [[11.0]], [[22.875]], [[65.0]], [[37.25]], [[50.25]], [[32.75]], [[35.5]], [[23.875]], [[51.0]], [[95.5]], [[66.5]], [[17.5]], [[63.25]], [[45.25]], [[21.125]], [[11.5625]], [[12.4375]], [[23.0]], [[14.375]], [[25.875]], [[36.5]]]}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[33.0]]], "params": {"weight": [[0.5234375]]}}, "language_model.model.layers.31.feed_forward.shared_expert.down_proj": {"inputs": [[[53.75]]], "outputs": [[[39.75]], [[1.914088313930279e+38]]], "params": {"weight": [[0.99609375]]}}, "language_model.model.layers.32.self_attn.qkv_proj": {"inputs": [[[53.25]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.32.self_attn.o_proj": {"inputs": [[[5.34375]]], "outputs": [[[96.0]], [[105.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.32.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.234375]], [[16.375]]]}, "language_model.model.layers.32.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.09375]]]}, "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.234375]]]}, "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.03125]]]}, "language_model.model.layers.32.self_attn.attn.impl.k_cache": {"inputs": [[[16.375]]]}, "language_model.model.layers.32.self_attn.attn.impl.v_cache": {"inputs": [[[6.09375]]]}, "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.4375]], [[16.375]], [[6.09375]]], "outputs": [[[5.34375]], [[1.0]]]}, "language_model.model.layers.32.feed_forward.gate_up_proj": {"inputs": [[[32.75]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.32.feed_forward.down_proj": {"inputs": [[[49.75]]], "outputs": [[[21.125]], [[79.5]]], "params": {"weight": [[0.490234375]]}}, "language_model.model.layers.33.self_attn.qkv_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.33.self_attn.o_proj": {"inputs": [[[5.25]]], "outputs": [[[97.0]], [[118.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.33.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1171875]], [[20.625]]]}, "language_model.model.layers.33.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.25]]]}, "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1171875]]]}, "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.34375]]]}, "language_model.model.layers.33.self_attn.attn.impl.k_cache": {"inputs": [[[20.625]]]}, "language_model.model.layers.33.self_attn.attn.impl.v_cache": {"inputs": [[[6.25]]]}, "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.25]], [[20.625]], [[6.25]]], "outputs": [[[5.25]], [[1.0]]]}, "language_model.model.layers.33.feed_forward.router": {"inputs": [[[64.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op": {"inputs": [[[64.0]]], "outputs": [[[10.25]], [[42.5]], [[49.0]], [[40.0]], [[44.5]], [[25.125]], [[78.0]], [[45.5]], [[88.0]], [[79.5]], [[55.5]], [[21.5]], [[62.75]], [[45.5]], [[27.75]], [[42.5]], [[26.125]], [[50.5]], [[53.75]], [[24.75]], [[22.875]], [[21.25]], [[30.875]], [[68.5]], [[31.25]], [[29.5]], [[35.5]], [[99.5]], [[89.0]], [[38.5]], [[51.25]], [[31.625]], [[45.0]], [[17.625]], [[93.0]], [[36.5]], [[28.0]], [[31.0]], [[15.0625]], [[94.5]], [[29.5]], [[19.5]], [[34.25]], [[88.5]], [[124.5]], [[35.75]], [[25.5]], [[83.0]], [[87.0]], [[65.0]], [[75.0]], [[34.5]], [[10.25]], [[38.75]], [[16.75]], [[110.5]], [[82.0]], [[43.25]], [[15.1875]], [[37.0]], [[82.5]], [[61.0]], [[45.5]], [[48.25]], [[72.0]], [[20.625]], [[75.5]], [[19.125]], [[18.0]], [[47.75]], [[33.5]], [[55.0]], [[54.5]], [[134.0]], [[58.75]], [[21.5]], [[82.0]], [[37.0]], [[32.5]], [[12.1875]], [[26.875]], [[18.875]], [[10.75]], [[68.5]], [[18.75]], [[26.125]], [[71.0]], [[47.25]], [[24.25]], [[85.0]], [[60.0]], [[17.25]], [[14.6875]], [[20.375]], [[40.5]], [[74.5]], [[60.25]], [[37.5]], [[52.25]], [[78.0]], [[85.0]], [[73.0]], [[61.5]], [[44.5]], [[70.5]], [[89.5]], [[21.625]], [[40.75]], [[22.0]], [[17.875]], [[10.375]], [[39.75]], [[166.0]], [[12.25]], [[34.5]], [[52.5]], [[20.375]], [[20.625]], [[33.25]], [[47.0]], [[64.0]], [[50.0]], [[37.25]], [[22.375]], [[8.125]], [[58.5]], [[28.625]], [[20.875]], [[47.0]]]}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.36328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[64.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.33.feed_forward.shared_expert.down_proj": {"inputs": [[[65.5]]], "outputs": [[[13.375]], [[2.6584559915698317e+38]]], "params": {"weight": [[0.435546875]]}}, "language_model.model.layers.34.self_attn.qkv_proj": {"inputs": [[[54.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.34.self_attn.o_proj": {"inputs": [[[5.90625]]], "outputs": [[[46.75]], [[80.5]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.34.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1875]], [[18.5]]]}, "language_model.model.layers.34.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.0]]]}, "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1875]]]}, "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.40625]]]}, "language_model.model.layers.34.self_attn.attn.impl.k_cache": {"inputs": [[[18.5]]]}, "language_model.model.layers.34.self_attn.attn.impl.v_cache": {"inputs": [[[8.0]]]}, "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.75]], [[18.5]], [[8.0]]], "outputs": [[[5.90625]], [[1.0]]]}, "language_model.model.layers.34.feed_forward.gate_up_proj": {"inputs": [[[31.125]]], "params": {"weight": [[0.376953125]]}}, "language_model.model.layers.34.feed_forward.down_proj": {"inputs": [[[54.0]]], "outputs": [[[30.375]], [[128.0]]], "params": {"weight": [[0.546875]]}}, "language_model.model.layers.35.self_attn.qkv_proj": {"inputs": [[[52.75]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.35.self_attn.o_proj": {"inputs": [[[9.6875]]], "outputs": [[[87.5]], [[93.0]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.35.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.89453125]], [[16.125]]]}, "language_model.model.layers.35.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.75]]]}, "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.89453125]]]}, "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[9.0625]]]}, "language_model.model.layers.35.self_attn.attn.impl.k_cache": {"inputs": [[[16.125]]]}, "language_model.model.layers.35.self_attn.attn.impl.v_cache": {"inputs": [[[11.75]]]}, "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.3125]], [[16.125]], [[11.5625]]], "outputs": [[[9.6875]], [[1.0]]]}, "language_model.model.layers.35.feed_forward.router": {"inputs": [[[64.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op": {"inputs": [[[64.0]]], "outputs": [[[13.3125]], [[127.5]], [[25.875]], [[18.875]], [[44.5]], [[137.0]], [[24.875]], [[54.5]], [[99.5]], [[103.0]], [[70.0]], [[105.0]], [[61.25]], [[88.0]], [[47.25]], [[104.0]], [[52.75]], [[115.5]], [[45.0]], [[63.75]], [[80.5]], [[56.25]], [[27.75]], [[25.25]], [[33.75]], [[54.0]], [[51.0]], [[91.0]], [[49.5]], [[29.0]], [[51.25]], [[84.5]], [[41.5]], [[45.25]], [[42.0]], [[58.5]], [[120.5]], [[44.25]], [[40.5]], [[42.0]], [[52.25]], [[34.75]], [[41.0]], [[19.125]], [[78.0]], [[92.0]], [[32.75]], [[15.25]], [[24.5]], [[110.5]], [[65.5]], [[30.0]], [[40.75]], [[85.0]], [[31.0]], [[27.5]], [[32.0]], [[94.5]], [[39.25]], [[64.5]], [[69.5]], [[71.5]], [[18.875]], [[55.5]], [[24.75]], [[41.25]], [[18.5]], [[129.0]], [[22.75]], [[21.375]], [[120.0]], [[61.5]], [[120.5]], [[224.0]], [[78.5]], [[31.375]], [[59.75]], [[37.0]], [[58.0]], [[85.5]], [[84.0]], [[70.5]], [[110.0]], [[22.125]], [[37.5]], [[45.0]], [[93.5]], [[54.5]], [[24.375]], [[17.375]], [[36.0]], [[150.0]], [[40.5]], [[55.0]], [[88.0]], [[131.0]], [[69.0]], [[77.0]], [[81.0]], [[100.0]], [[34.0]], [[126.0]], [[17.25]], [[53.25]], [[28.125]], [[42.75]], [[87.0]], [[46.25]], [[25.75]], [[117.5]], [[50.0]], [[73.0]], [[60.0]], [[52.25]], [[100.0]], [[45.5]], [[71.5]], [[51.5]], [[32.75]], [[45.0]], [[50.75]], [[96.5]], [[24.75]], [[54.0]], [[98.0]], [[55.5]], [[45.5]], [[24.25]], [[28.625]]]}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.380859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.365234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[64.0]]], "params": {"weight": [[0.5078125]]}}, "language_model.model.layers.35.feed_forward.shared_expert.down_proj": {"inputs": [[[78.0]]], "outputs": [[[13.125]], [[1.6698426697048006e+37]]], "params": {"weight": [[0.6171875]]}}, "language_model.model.layers.36.self_attn.qkv_proj": {"inputs": [[[56.75]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.36.self_attn.o_proj": {"inputs": [[[7.53125]]], "outputs": [[[27.75]], [[43.0]]], "params": {"weight": [[0.3984375]]}}, "language_model.model.layers.36.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1796875]], [[20.375]]]}, "language_model.model.layers.36.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.375]]]}, "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1796875]]]}, "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.90625]]]}, "language_model.model.layers.36.self_attn.attn.impl.k_cache": {"inputs": [[[20.5]]]}, "language_model.model.layers.36.self_attn.attn.impl.v_cache": {"inputs": [[[9.375]]]}, "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.0625]], [[20.5]], [[9.375]]], "outputs": [[[7.53125]], [[1.0]]]}, "language_model.model.layers.36.feed_forward.gate_up_proj": {"inputs": [[[29.625]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.36.feed_forward.down_proj": {"inputs": [[[60.5]]], "outputs": [[[38.25]], [[76.5]]], "params": {"weight": [[0.9453125]]}}, "language_model.model.layers.37.self_attn.qkv_proj": {"inputs": [[[50.25]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.37.self_attn.o_proj": {"inputs": [[[9.625]]], "outputs": [[[102.0]], [[105.0]]], "params": {"weight": [[0.984375]]}}, "language_model.model.layers.37.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1796875]], [[12.3125]]]}, "language_model.model.layers.37.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.8125]]]}, "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1796875]]]}, "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[12.0]]]}, "language_model.model.layers.37.self_attn.attn.impl.k_cache": {"inputs": [[[12.3125]]]}, "language_model.model.layers.37.self_attn.attn.impl.v_cache": {"inputs": [[[12.8125]]]}, "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.625]], [[12.3125]], [[12.8125]]], "outputs": [[[9.625]], [[1.0]]]}, "language_model.model.layers.37.feed_forward.router": {"inputs": [[[70.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op": {"inputs": [[[70.0]]], "outputs": [[[10.625]], [[68.5]], [[97.0]], [[67.0]], [[53.25]], [[35.5]], [[62.25]], [[64.5]], [[66.0]], [[36.75]], [[56.25]], [[30.625]], [[29.625]], [[28.125]], [[35.25]], [[74.0]], [[40.5]], [[47.25]], [[49.25]], [[80.0]], [[52.75]], [[98.0]], [[63.75]], [[78.0]], [[54.5]], [[103.0]], [[51.25]], [[75.5]], [[77.0]], [[118.5]], [[63.25]], [[31.5]], [[105.5]], [[26.875]], [[23.375]], [[68.5]], [[79.0]], [[86.0]], [[130.0]], [[83.0]], [[93.5]], [[108.5]], [[25.625]], [[40.0]], [[50.5]], [[34.0]], [[102.0]], [[112.0]], [[21.75]], [[67.0]], [[14.5]], [[36.25]], [[54.5]], [[81.0]], [[44.0]], [[31.125]], [[76.0]], [[119.5]], [[41.25]], [[83.5]], [[124.5]], [[73.0]], [[41.0]], [[36.0]], [[28.625]], [[47.75]], [[31.375]], [[56.75]], [[77.5]], [[76.0]], [[29.375]], [[62.0]], [[32.25]], [[45.75]], [[21.625]], [[28.875]], [[72.0]], [[145.0]], [[27.375]], [[52.0]], [[82.5]], [[105.0]], [[57.0]], [[89.0]], [[30.125]], [[52.5]], [[26.875]], [[106.0]], [[110.5]], [[119.0]], [[155.0]], [[46.5]], [[74.0]], [[41.5]], [[58.75]], [[39.5]], [[81.5]], [[36.25]], [[24.625]], [[101.0]], [[106.0]], [[165.0]], [[53.5]], [[89.0]], [[34.5]], [[77.5]], [[71.5]], [[74.5]], [[22.625]], [[80.0]], [[28.75]], [[92.0]], [[58.5]], [[105.5]], [[86.0]], [[40.0]], [[99.0]], [[65.0]], [[99.0]], [[50.25]], [[84.0]], [[26.125]], [[41.0]], [[71.5]], [[96.0]], [[98.0]], [[25.5]], [[45.25]], [[67.5]]]}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.431640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[70.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.37.feed_forward.shared_expert.down_proj": {"inputs": [[[107.0]]], "outputs": [[[18.125]], [[1.914088313930279e+38]]], "params": {"weight": [[0.60546875]]}}, "language_model.model.layers.38.self_attn.qkv_proj": {"inputs": [[[47.25]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.38.self_attn.o_proj": {"inputs": [[[10.0]]], "outputs": [[[26.75]], [[56.0]]], "params": {"weight": [[0.90625]]}}, "language_model.model.layers.38.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.421875]], [[12.0625]]]}, "language_model.model.layers.38.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.5625]]]}, "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.421875]]]}, "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[10.0]]]}, "language_model.model.layers.38.self_attn.attn.impl.k_cache": {"inputs": [[[12.0625]]]}, "language_model.model.layers.38.self_attn.attn.impl.v_cache": {"inputs": [[[12.5625]]]}, "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[17.875]], [[12.0625]], [[12.5625]]], "outputs": [[[10.0]], [[1.0]]]}, "language_model.model.layers.38.feed_forward.gate_up_proj": {"inputs": [[[31.125]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.38.feed_forward.down_proj": {"inputs": [[[85.5]]], "outputs": [[[24.75]], [[193.0]]], "params": {"weight": [[0.6953125]]}}, "language_model.model.layers.39.self_attn.qkv_proj": {"inputs": [[[49.75]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.39.self_attn.o_proj": {"inputs": [[[7.40625]]], "outputs": [[[129.0]], [[134.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.39.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.109375]], [[10.625]]]}, "language_model.model.layers.39.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.4375]]]}, "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.109375]]]}, "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[10.0]]]}, "language_model.model.layers.39.self_attn.attn.impl.k_cache": {"inputs": [[[10.625]]]}, "language_model.model.layers.39.self_attn.attn.impl.v_cache": {"inputs": [[[12.4375]]]}, "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.5]], [[10.625]], [[12.4375]]], "outputs": [[[7.40625]], [[1.0]]]}, "language_model.model.layers.39.feed_forward.router": {"inputs": [[[61.5]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op": {"inputs": [[[61.5]]], "outputs": [[[14.4375]], [[71.0]], [[40.0]], [[72.0]], [[79.0]], [[71.5]], [[48.75]], [[44.5]], [[124.0]], [[57.5]], [[60.5]], [[95.0]], [[48.5]], [[111.5]], [[94.0]], [[58.25]], [[52.5]], [[91.5]], [[82.0]], [[49.5]], [[73.0]], [[42.0]], [[114.5]], [[35.25]], [[91.5]], [[66.0]], [[74.5]], [[67.5]], [[118.5]], [[101.5]], [[47.0]], [[56.25]], [[103.0]], [[28.5]], [[34.0]], [[102.5]], [[55.25]], [[94.5]], [[68.5]], [[123.0]], [[120.0]], [[86.5]], [[45.75]], [[123.5]], [[77.5]], [[56.25]], [[45.25]], [[52.25]], [[34.0]], [[98.5]], [[105.5]], [[103.0]], [[143.0]], [[48.25]], [[43.75]], [[68.0]], [[109.0]], [[33.5]], [[100.5]], [[43.75]], [[53.75]], [[118.5]], [[117.5]], [[73.0]], [[21.125]], [[45.75]], [[47.5]], [[59.25]], [[112.0]], [[90.0]], [[101.0]], [[68.0]], [[23.625]], [[80.5]], [[71.0]], [[74.5]], [[61.0]], [[95.0]], [[119.5]], [[132.0]], [[42.5]], [[50.0]], [[108.5]], [[68.5]], [[54.5]], [[107.5]], [[53.25]], [[167.0]], [[52.25]], [[94.0]], [[73.5]], [[48.75]], [[86.0]], [[46.0]], [[40.0]], [[67.0]], [[49.0]], [[76.5]], [[92.5]], [[75.0]], [[47.0]], [[54.0]], [[55.25]], [[99.0]], [[102.5]], [[62.25]], [[148.0]], [[92.0]], [[121.0]], [[52.75]], [[53.5]], [[39.5]], [[65.0]], [[22.75]], [[73.5]], [[75.0]], [[142.0]], [[25.5]], [[35.25]], [[129.0]], [[100.0]], [[32.0]], [[58.75]], [[89.5]], [[41.5]], [[79.0]], [[33.75]], [[87.0]], [[76.5]]]}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.3046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[61.5]]], "params": {"weight": [[0.474609375]]}}, "language_model.model.layers.39.feed_forward.shared_expert.down_proj": {"inputs": [[[153.0]]], "outputs": [[[21.5]], [[1.914088313930279e+38]]], "params": {"weight": [[0.4453125]]}}, "language_model.model.layers.40.self_attn.qkv_proj": {"inputs": [[[46.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.40.self_attn.o_proj": {"inputs": [[[13.25]]], "outputs": [[[93.0]], [[90.0]]], "params": {"weight": [[0.48828125]]}}, "language_model.model.layers.40.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.140625]], [[11.25]]]}, "language_model.model.layers.40.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[18.0]]]}, "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.140625]]]}, "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[12.9375]]]}, "language_model.model.layers.40.self_attn.attn.impl.k_cache": {"inputs": [[[11.6875]]]}, "language_model.model.layers.40.self_attn.attn.impl.v_cache": {"inputs": [[[18.0]]]}, "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.375]], [[11.6875]], [[18.0]]], "outputs": [[[13.25]], [[1.0]]]}, "language_model.model.layers.40.feed_forward.gate_up_proj": {"inputs": [[[40.75]]], "params": {"weight": [[0.32421875]]}}, "language_model.model.layers.40.feed_forward.down_proj": {"inputs": [[[152.0]]], "outputs": [[[22.625]], [[146.0]]], "params": {"weight": [[0.62109375]]}}, "language_model.model.layers.41.self_attn.qkv_proj": {"inputs": [[[33.75]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.41.self_attn.o_proj": {"inputs": [[[12.6875]]], "outputs": [[[159.0]], [[164.0]]], "params": {"weight": [[1.1875]]}}, "language_model.model.layers.41.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.2578125]], [[18.5]]]}, "language_model.model.layers.41.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.8125]]]}, "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.2578125]]]}, "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[11.1875]]]}, "language_model.model.layers.41.self_attn.attn.impl.k_cache": {"inputs": [[[18.5]]]}, "language_model.model.layers.41.self_attn.attn.impl.v_cache": {"inputs": [[[12.8125]]]}, "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.5]], [[18.5]], [[12.8125]]], "outputs": [[[12.6875]], [[1.0]]]}, "language_model.model.layers.41.feed_forward.router": {"inputs": [[[68.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op": {"inputs": [[[68.0]]], "outputs": [[[17.375]], [[70.0]], [[40.0]], [[41.75]], [[125.0]], [[41.5]], [[48.5]], [[109.0]], [[65.5]], [[57.0]], [[114.0]], [[100.5]], [[40.75]], [[63.0]], [[111.0]], [[126.5]], [[106.5]], [[132.0]], [[130.0]], [[121.5]], [[56.0]], [[98.5]], [[47.25]], [[104.5]], [[133.0]], [[146.0]], [[158.0]], [[114.0]], [[70.5]], [[59.0]], [[141.0]], [[73.0]], [[113.0]], [[180.0]], [[46.25]], [[200.0]], [[116.0]], [[48.75]], [[121.5]], [[48.25]], [[57.5]], [[84.0]], [[169.0]], [[58.25]], [[40.25]], [[46.0]], [[79.5]], [[109.5]], [[87.5]], [[178.0]], [[118.5]], [[172.0]], [[112.0]], [[100.0]], [[84.0]], [[104.5]], [[126.0]], [[167.0]], [[131.0]], [[99.0]], [[50.25]], [[124.5]], [[145.0]], [[61.5]], [[69.0]], [[112.0]], [[77.0]], [[98.0]], [[73.0]], [[32.25]], [[85.0]], [[47.75]], [[47.25]], [[99.5]], [[27.75]], [[136.0]], [[114.5]], [[77.5]], [[80.0]], [[52.0]], [[120.0]], [[121.0]], [[118.5]], [[83.5]], [[73.5]], [[91.5]], [[49.5]], [[83.0]], [[87.0]], [[56.25]], [[107.5]], [[50.25]], [[97.0]], [[102.5]], [[74.0]], [[45.0]], [[66.0]], [[75.5]], [[66.5]], [[85.0]], [[63.0]], [[132.0]], [[86.0]], [[105.0]], [[45.0]], [[58.75]], [[103.5]], [[68.5]], [[223.0]], [[26.125]], [[75.0]], [[155.0]], [[134.0]], [[89.0]], [[70.5]], [[82.5]], [[118.0]], [[96.0]], [[107.0]], [[91.0]], [[74.5]], [[113.5]], [[80.0]], [[173.0]], [[59.0]], [[47.5]], [[63.75]], [[110.0]], [[165.0]]]}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.365234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.435546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.41015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[68.0]]], "params": {"weight": [[0.36328125]]}}, "language_model.model.layers.41.feed_forward.shared_expert.down_proj": {"inputs": [[[256.0]]], "outputs": [[[32.0]], [[1.914088313930279e+38]]], "params": {"weight": [[0.65234375]]}}, "language_model.model.layers.42.self_attn.qkv_proj": {"inputs": [[[38.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.42.self_attn.o_proj": {"inputs": [[[14.375]]], "outputs": [[[50.5]], [[134.0]]], "params": {"weight": [[0.671875]]}}, "language_model.model.layers.42.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.2421875]], [[14.1875]]]}, "language_model.model.layers.42.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[14.6875]]]}, "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.2421875]]]}, "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[11.4375]]]}, "language_model.model.layers.42.self_attn.attn.impl.k_cache": {"inputs": [[[14.1875]]]}, "language_model.model.layers.42.self_attn.attn.impl.v_cache": {"inputs": [[[15.5625]]]}, "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[16.75]], [[14.1875]], [[15.5625]]], "outputs": [[[14.375]], [[1.0]]]}, "language_model.model.layers.42.feed_forward.gate_up_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.42.feed_forward.down_proj": {"inputs": [[[180.0]]], "outputs": [[[59.5]], [[338.0]]], "params": {"weight": [[0.55859375]]}}, "language_model.model.layers.43.self_attn.qkv_proj": {"inputs": [[[45.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.43.self_attn.o_proj": {"inputs": [[[13.8125]]], "outputs": [[[94.5]], [[232.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.43.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1796875]], [[10.1875]]]}, "language_model.model.layers.43.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[27.25]]]}, "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1796875]]]}, "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[16.375]]]}, "language_model.model.layers.43.self_attn.attn.impl.k_cache": {"inputs": [[[10.1875]]]}, "language_model.model.layers.43.self_attn.attn.impl.v_cache": {"inputs": [[[27.25]]]}, "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.875]], [[10.1875]], [[27.25]]], "outputs": [[[13.25]], [[1.0]]]}, "language_model.model.layers.43.feed_forward.router": {"inputs": [[[68.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op": {"inputs": [[[68.0]]], "outputs": [[[22.375]], [[100.5]], [[113.0]], [[64.0]], [[90.5]], [[149.0]], [[37.5]], [[58.0]], [[156.0]], [[67.0]], [[157.0]], [[89.5]], [[115.0]], [[64.5]], [[81.5]], [[129.0]], [[106.5]], [[81.5]], [[116.5]], [[141.0]], [[58.75]], [[254.0]], [[145.0]], [[125.0]], [[102.0]], [[107.5]], [[153.0]], [[91.0]], [[74.0]], [[106.5]], [[57.25]], [[41.5]], [[76.5]], [[146.0]], [[139.0]], [[102.0]], [[117.5]], [[75.0]], [[166.0]], [[52.5]], [[116.5]], [[108.5]], [[111.0]], [[148.0]], [[46.0]], [[192.0]], [[99.0]], [[72.5]], [[32.25]], [[93.5]], [[91.0]], [[55.5]], [[162.0]], [[136.0]], [[85.5]], [[100.5]], [[91.5]], [[43.5]], [[58.25]], [[42.75]], [[76.5]], [[40.5]], [[58.0]], [[55.0]], [[182.0]], [[210.0]], [[91.5]], [[93.0]], [[58.5]], [[344.0]], [[122.0]], [[136.0]], [[84.0]], [[89.5]], [[99.0]], [[50.25]], [[59.75]], [[170.0]], [[66.0]], [[100.5]], [[64.0]], [[114.0]], [[91.0]], [[132.0]], [[86.5]], [[71.0]], [[174.0]], [[84.5]], [[62.25]], [[81.0]], [[53.0]], [[100.0]], [[50.5]], [[81.5]], [[45.5]], [[44.25]], [[58.25]], [[44.0]], [[132.0]], [[58.5]], [[175.0]], [[91.0]], [[54.5]], [[71.5]], [[74.0]], [[114.5]], [[73.0]], [[89.5]], [[99.5]], [[61.75]], [[81.0]], [[192.0]], [[86.0]], [[138.0]], [[83.5]], [[33.75]], [[72.0]], [[86.5]], [[90.0]], [[84.0]], [[52.75]], [[47.25]], [[139.0]], [[130.0]], [[112.0]], [[66.0]], [[123.5]], [[100.0]], [[59.75]]]}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.3828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.42578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[68.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.43.feed_forward.shared_expert.down_proj": {"inputs": [[[207.0]]], "outputs": [[[33.25]], [[1.914088313930279e+38]]], "params": {"weight": [[0.625]]}}, "language_model.model.layers.44.self_attn.qkv_proj": {"inputs": [[[28.625]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.44.self_attn.o_proj": {"inputs": [[[17.75]]], "outputs": [[[68.0]], [[153.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.44.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.3125]], [[11.875]]]}, "language_model.model.layers.44.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[39.5]]]}, "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.3125]]]}, "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[16.5]]]}, "language_model.model.layers.44.self_attn.attn.impl.k_cache": {"inputs": [[[11.875]]]}, "language_model.model.layers.44.self_attn.attn.impl.v_cache": {"inputs": [[[39.5]]]}, "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.875]], [[11.875]], [[39.5]]], "outputs": [[[17.75]], [[1.0]]]}, "language_model.model.layers.44.feed_forward.gate_up_proj": {"inputs": [[[41.5]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.44.feed_forward.down_proj": {"inputs": [[[249.0]]], "outputs": [[[63.25]], [[191.0]]], "params": {"weight": [[0.63671875]]}}, "language_model.model.layers.45.self_attn.qkv_proj": {"inputs": [[[27.375]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.45.self_attn.o_proj": {"inputs": [[[15.6875]]], "outputs": [[[169.0]], [[220.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.45.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.65625]], [[18.375]]]}, "language_model.model.layers.45.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[22.125]]]}, "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.65625]]]}, "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[13.0]]]}, "language_model.model.layers.45.self_attn.attn.impl.k_cache": {"inputs": [[[18.375]]]}, "language_model.model.layers.45.self_attn.attn.impl.v_cache": {"inputs": [[[22.125]]]}, "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[20.375]], [[18.375]], [[22.125]]], "outputs": [[[15.6875]], [[1.0]]]}, "language_model.model.layers.45.feed_forward.router": {"inputs": [[[51.75]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op": {"inputs": [[[51.75]]], "outputs": [[[11.9375]], [[55.25]], [[32.0]], [[37.0]], [[55.5]], [[0.0]], [[28.75]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[45.75]], [[30.125]], [[30.25]], [[38.0]], [[0.0]], [[0.0]], [[0.0]], [[44.25]], [[0.0]], [[0.0]], [[0.0]], [[28.375]], [[0.0]], [[44.5]], [[77.5]], [[18.375]], [[46.75]], [[41.0]], [[0.0]], [[15.1875]], [[27.75]], [[31.875]], [[79.5]], [[59.75]], [[23.625]], [[0.0]], [[33.75]], [[15.375]], [[83.5]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[92.0]], [[0.0]], [[89.0]], [[0.0]], [[0.0]], [[0.0]], [[93.5]], [[0.0]], [[112.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[46.25]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[183.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[50.0]], [[50.25]], [[85.5]], [[23.625]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[58.25]], [[0.0]], [[40.25]], [[43.0]], [[73.5]], [[53.75]], [[0.0]], [[0.0]], [[20.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[20.25]], [[9.324138683375338e-18]], [[48.0]], [[0.0]], [[69.5]], [[0.0]], [[0.0]]]}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.73828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.35546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[51.75]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.45.feed_forward.shared_expert.down_proj": {"inputs": [[[218.0]]], "outputs": [[[31.75]], [[1.914088313930279e+38]]], "params": {"weight": [[0.5546875]]}}, "language_model.model.layers.46.self_attn.qkv_proj": {"inputs": [[[26.75]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.46.self_attn.o_proj": {"inputs": [[[14.1875]]], "outputs": [[[101.0]], [[188.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.46.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.34375]], [[12.375]]]}, "language_model.model.layers.46.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[15.4375]]]}, "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.34375]]]}, "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[12.8125]]]}, "language_model.model.layers.46.self_attn.attn.impl.k_cache": {"inputs": [[[12.375]]]}, "language_model.model.layers.46.self_attn.attn.impl.v_cache": {"inputs": [[[15.4375]]]}, "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[18.375]], [[12.375]], [[15.4375]]], "outputs": [[[14.1875]], [[1.0]]]}, "language_model.model.layers.46.feed_forward.gate_up_proj": {"inputs": [[[40.5]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.46.feed_forward.down_proj": {"inputs": [[[384.0]]], "outputs": [[[1688.0]], [[2400.0]]], "params": {"weight": [[0.59375]]}}, "language_model.model.layers.47.self_attn.qkv_proj": {"inputs": [[[19.875]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.47.self_attn.o_proj": {"inputs": [[[3.328125]]], "outputs": [[[98.0]], [[126.5]]], "params": {"weight": [[0.3671875]]}}, "language_model.model.layers.47.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.53515625]], [[4.8125]]]}, "language_model.model.layers.47.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.90625]]]}, "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.53515625]]]}, "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.6875]]]}, "language_model.model.layers.47.self_attn.attn.impl.k_cache": {"inputs": [[[4.8125]]]}, "language_model.model.layers.47.self_attn.attn.impl.v_cache": {"inputs": [[[4.90625]]]}, "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[6.6875]], [[4.8125]], [[4.90625]]], "outputs": [[[2.640625]], [[1.0]]]}, "language_model.model.layers.47.feed_forward.router": {"inputs": [[[33.5]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op": {"inputs": [[[33.5]]], "outputs": [[[142.0]], [[55.75]], [[109.5]], [[62.5]], [[88.5]], [[66.5]], [[30.125]], [[41.0]], [[49.75]], [[118.0]], [[270.0]], [[49.25]], [[93.0]], [[103.0]], [[57.25]], [[83.5]], [[26.875]], [[114.0]], [[104.5]], [[51.0]], [[117.5]], [[72.0]], [[91.0]], [[47.75]], [[59.5]], [[214.0]], [[249.0]], [[27.875]], [[28.125]], [[41.0]], [[55.5]], [[44.5]], [[153.0]], [[23.5]], [[40.0]], [[71.0]], [[34.75]], [[29.375]], [[23.375]], [[247.0]], [[90.0]], [[29.0]], [[101.5]], [[131.0]], [[33.5]], [[39.25]], [[86.0]], [[128.0]], [[46.5]], [[24.125]], [[107.0]], [[140.0]], [[258.0]], [[51.5]], [[29.5]], [[47.0]], [[63.5]], [[99.0]], [[25.125]], [[30.375]], [[125.5]], [[50.75]], [[66.5]], [[137.0]], [[132.0]], [[39.5]], [[36.0]], [[89.5]], [[186.0]], [[30.375]], [[168.0]], [[256.0]], [[28.0]], [[32.5]], [[41.25]], [[95.0]], [[19.125]], [[51.0]], [[44.25]], [[142.0]], [[68.0]], [[77.5]], [[344.0]], [[516.0]], [[77.0]], [[35.75]], [[99.5]], [[110.0]], [[43.5]], [[30.125]], [[50.5]], [[119.5]], [[245.0]], [[310.0]], [[27.5]], [[38.5]], [[40.5]], [[64.5]], [[53.5]], [[32.75]], [[51.0]], [[81.5]], [[45.25]], [[102.0]], [[44.25]], [[105.0]], [[38.0]], [[76.5]], [[338.0]], [[33.0]], [[92.5]], [[103.5]], [[163.0]], [[44.75]], [[82.5]], [[87.0]], [[52.5]], [[74.5]], [[211.0]], [[43.5]], [[186.0]], [[111.5]], [[84.0]], [[101.5]], [[26.125]], [[62.75]], [[123.0]], [[158.0]], [[322.0]]]}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.48046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.361328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.4921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.6875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.81640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.69140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.66015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.66015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.78125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.4453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.83984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.40234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.404296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.5703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.5859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.498046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.6875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.83203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.6015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.79296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.48828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.36328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.376953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.427734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.41015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.45703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.65625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.53515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.44921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.35546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.7578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.48828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.6640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.82421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.46875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.57421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.50390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.48828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.65625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.63671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.57421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.63671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.69921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.7421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.75]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.73828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.40234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.50390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.5859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.67578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.5078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.40234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.458984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.6796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.38671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.6875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.58203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.4375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.53125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.66796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.70703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.439453125]]}}, "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[33.5]]], "params": {"weight": [[0.40234375]]}}, "language_model.model.layers.47.feed_forward.shared_expert.down_proj": {"inputs": [[[484.0]]], "outputs": [[[454.0]], [[1.914088313930279e+38]]], "params": {"weight": [[0.56640625]]}}, "language_model.lm_head": {"inputs": [[[52.0]]], "params": {"weight": [[1.0859375]]}}}} \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_2_8.npz b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_2_8.npz new file mode 100644 index 000000000000..f40812c6fb38 Binary files /dev/null and b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_2_8.npz differ diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_2_8_mod_list.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_2_8_mod_list.json new file mode 100644 index 000000000000..928cd9e2a2c5 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_2_8_mod_list.json @@ -0,0 +1,6935 @@ +[ + "vision_model.patch_embedding.linear", + "vision_model.model.layers.0.self_attn.qkv_proj", + "vision_model.model.layers.0.self_attn.o_proj", + "vision_model.model.layers.0.mlp.fc1", + "vision_model.model.layers.0.mlp.fc2", + "vision_model.model.layers.1.self_attn.qkv_proj", + "vision_model.model.layers.1.self_attn.o_proj", + "vision_model.model.layers.1.mlp.fc1", + "vision_model.model.layers.1.mlp.fc2", + "vision_model.model.layers.2.self_attn.qkv_proj", + "vision_model.model.layers.2.self_attn.o_proj", + "vision_model.model.layers.2.mlp.fc1", + "vision_model.model.layers.2.mlp.fc2", + "vision_model.model.layers.3.self_attn.qkv_proj", + "vision_model.model.layers.3.self_attn.o_proj", + "vision_model.model.layers.3.mlp.fc1", + "vision_model.model.layers.3.mlp.fc2", + "vision_model.model.layers.4.self_attn.qkv_proj", + "vision_model.model.layers.4.self_attn.o_proj", + "vision_model.model.layers.4.mlp.fc1", + "vision_model.model.layers.4.mlp.fc2", + "vision_model.model.layers.5.self_attn.qkv_proj", + "vision_model.model.layers.5.self_attn.o_proj", + "vision_model.model.layers.5.mlp.fc1", + "vision_model.model.layers.5.mlp.fc2", + "vision_model.model.layers.6.self_attn.qkv_proj", + "vision_model.model.layers.6.self_attn.o_proj", + "vision_model.model.layers.6.mlp.fc1", + "vision_model.model.layers.6.mlp.fc2", + "vision_model.model.layers.7.self_attn.qkv_proj", + "vision_model.model.layers.7.self_attn.o_proj", + "vision_model.model.layers.7.mlp.fc1", + "vision_model.model.layers.7.mlp.fc2", + "vision_model.model.layers.8.self_attn.qkv_proj", + "vision_model.model.layers.8.self_attn.o_proj", + "vision_model.model.layers.8.mlp.fc1", + "vision_model.model.layers.8.mlp.fc2", + "vision_model.model.layers.9.self_attn.qkv_proj", + "vision_model.model.layers.9.self_attn.o_proj", + "vision_model.model.layers.9.mlp.fc1", + "vision_model.model.layers.9.mlp.fc2", + "vision_model.model.layers.10.self_attn.qkv_proj", + "vision_model.model.layers.10.self_attn.o_proj", + "vision_model.model.layers.10.mlp.fc1", + "vision_model.model.layers.10.mlp.fc2", + "vision_model.model.layers.11.self_attn.qkv_proj", + "vision_model.model.layers.11.self_attn.o_proj", + "vision_model.model.layers.11.mlp.fc1", + "vision_model.model.layers.11.mlp.fc2", + "vision_model.model.layers.12.self_attn.qkv_proj", + "vision_model.model.layers.12.self_attn.o_proj", + "vision_model.model.layers.12.mlp.fc1", + "vision_model.model.layers.12.mlp.fc2", + "vision_model.model.layers.13.self_attn.qkv_proj", + "vision_model.model.layers.13.self_attn.o_proj", + "vision_model.model.layers.13.mlp.fc1", + "vision_model.model.layers.13.mlp.fc2", + "vision_model.model.layers.14.self_attn.qkv_proj", + "vision_model.model.layers.14.self_attn.o_proj", + "vision_model.model.layers.14.mlp.fc1", + "vision_model.model.layers.14.mlp.fc2", + "vision_model.model.layers.15.self_attn.qkv_proj", + "vision_model.model.layers.15.self_attn.o_proj", + "vision_model.model.layers.15.mlp.fc1", + "vision_model.model.layers.15.mlp.fc2", + "vision_model.model.layers.16.self_attn.qkv_proj", + "vision_model.model.layers.16.self_attn.o_proj", + "vision_model.model.layers.16.mlp.fc1", + "vision_model.model.layers.16.mlp.fc2", + "vision_model.model.layers.17.self_attn.qkv_proj", + "vision_model.model.layers.17.self_attn.o_proj", + "vision_model.model.layers.17.mlp.fc1", + "vision_model.model.layers.17.mlp.fc2", + "vision_model.model.layers.18.self_attn.qkv_proj", + "vision_model.model.layers.18.self_attn.o_proj", + "vision_model.model.layers.18.mlp.fc1", + "vision_model.model.layers.18.mlp.fc2", + "vision_model.model.layers.19.self_attn.qkv_proj", + "vision_model.model.layers.19.self_attn.o_proj", + "vision_model.model.layers.19.mlp.fc1", + "vision_model.model.layers.19.mlp.fc2", + "vision_model.model.layers.20.self_attn.qkv_proj", + "vision_model.model.layers.20.self_attn.o_proj", + "vision_model.model.layers.20.mlp.fc1", + "vision_model.model.layers.20.mlp.fc2", + "vision_model.model.layers.21.self_attn.qkv_proj", + "vision_model.model.layers.21.self_attn.o_proj", + "vision_model.model.layers.21.mlp.fc1", + "vision_model.model.layers.21.mlp.fc2", + "vision_model.model.layers.22.self_attn.qkv_proj", + "vision_model.model.layers.22.self_attn.o_proj", + "vision_model.model.layers.22.mlp.fc1", + "vision_model.model.layers.22.mlp.fc2", + "vision_model.model.layers.23.self_attn.qkv_proj", + "vision_model.model.layers.23.self_attn.o_proj", + "vision_model.model.layers.23.mlp.fc1", + "vision_model.model.layers.23.mlp.fc2", + "vision_model.model.layers.24.self_attn.qkv_proj", + "vision_model.model.layers.24.self_attn.o_proj", + "vision_model.model.layers.24.mlp.fc1", + "vision_model.model.layers.24.mlp.fc2", + "vision_model.model.layers.25.self_attn.qkv_proj", + "vision_model.model.layers.25.self_attn.o_proj", + "vision_model.model.layers.25.mlp.fc1", + "vision_model.model.layers.25.mlp.fc2", + "vision_model.model.layers.26.self_attn.qkv_proj", + "vision_model.model.layers.26.self_attn.o_proj", + "vision_model.model.layers.26.mlp.fc1", + "vision_model.model.layers.26.mlp.fc2", + "vision_model.model.layers.27.self_attn.qkv_proj", + "vision_model.model.layers.27.self_attn.o_proj", + "vision_model.model.layers.27.mlp.fc1", + "vision_model.model.layers.27.mlp.fc2", + "vision_model.model.layers.28.self_attn.qkv_proj", + "vision_model.model.layers.28.self_attn.o_proj", + "vision_model.model.layers.28.mlp.fc1", + "vision_model.model.layers.28.mlp.fc2", + "vision_model.model.layers.29.self_attn.qkv_proj", + "vision_model.model.layers.29.self_attn.o_proj", + "vision_model.model.layers.29.mlp.fc1", + "vision_model.model.layers.29.mlp.fc2", + "vision_model.model.layers.30.self_attn.qkv_proj", + "vision_model.model.layers.30.self_attn.o_proj", + "vision_model.model.layers.30.mlp.fc1", + "vision_model.model.layers.30.mlp.fc2", + "vision_model.model.layers.31.self_attn.qkv_proj", + "vision_model.model.layers.31.self_attn.o_proj", + "vision_model.model.layers.31.mlp.fc1", + "vision_model.model.layers.31.mlp.fc2", + "vision_model.model.layers.32.self_attn.qkv_proj", + "vision_model.model.layers.32.self_attn.o_proj", + "vision_model.model.layers.32.mlp.fc1", + "vision_model.model.layers.32.mlp.fc2", + "vision_model.model.layers.33.self_attn.qkv_proj", + "vision_model.model.layers.33.self_attn.o_proj", + "vision_model.model.layers.33.mlp.fc1", + "vision_model.model.layers.33.mlp.fc2", + "vision_model.vision_adapter.mlp.fc1", + "vision_model.vision_adapter.mlp.fc2", + "multi_modal_projector.linear_1", + "language_model.model.layers.0.self_attn.qkv_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.0.self_attn.attn.impl.softmax", + "language_model.model.layers.0.self_attn.attn.impl.matmul_av", + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.0.self_attn.attn.impl.k_cache", + "language_model.model.layers.0.self_attn.attn.impl.v_cache", + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.0.feed_forward.gate_up_proj", + "language_model.model.layers.0.feed_forward.down_proj", + "language_model.model.layers.1.self_attn.qkv_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.1.self_attn.attn.impl.softmax", + "language_model.model.layers.1.self_attn.attn.impl.matmul_av", + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.1.self_attn.attn.impl.k_cache", + "language_model.model.layers.1.self_attn.attn.impl.v_cache", + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.1.feed_forward.router", + "language_model.model.layers.1.feed_forward.experts", + "language_model.model.layers.1.feed_forward.experts.moe_op", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.1.feed_forward.shared_expert.down_proj", + "language_model.model.layers.2.self_attn.qkv_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.2.self_attn.attn.impl.softmax", + "language_model.model.layers.2.self_attn.attn.impl.matmul_av", + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.2.self_attn.attn.impl.k_cache", + "language_model.model.layers.2.self_attn.attn.impl.v_cache", + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.2.feed_forward.gate_up_proj", + "language_model.model.layers.2.feed_forward.down_proj", + "language_model.model.layers.3.self_attn.qkv_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.3.self_attn.attn.impl.softmax", + "language_model.model.layers.3.self_attn.attn.impl.matmul_av", + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.3.self_attn.attn.impl.k_cache", + "language_model.model.layers.3.self_attn.attn.impl.v_cache", + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.3.feed_forward.router", + "language_model.model.layers.3.feed_forward.experts", + "language_model.model.layers.3.feed_forward.experts.moe_op", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.3.feed_forward.shared_expert.down_proj", + "language_model.model.layers.4.self_attn.qkv_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.4.self_attn.attn.impl.softmax", + "language_model.model.layers.4.self_attn.attn.impl.matmul_av", + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.4.self_attn.attn.impl.k_cache", + "language_model.model.layers.4.self_attn.attn.impl.v_cache", + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.4.feed_forward.gate_up_proj", + "language_model.model.layers.4.feed_forward.down_proj", + "language_model.model.layers.5.self_attn.qkv_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.5.self_attn.attn.impl.softmax", + "language_model.model.layers.5.self_attn.attn.impl.matmul_av", + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.5.self_attn.attn.impl.k_cache", + "language_model.model.layers.5.self_attn.attn.impl.v_cache", + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.5.feed_forward.router", + "language_model.model.layers.5.feed_forward.experts", + "language_model.model.layers.5.feed_forward.experts.moe_op", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.5.feed_forward.shared_expert.down_proj", + "language_model.model.layers.6.self_attn.qkv_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.6.self_attn.attn.impl.softmax", + "language_model.model.layers.6.self_attn.attn.impl.matmul_av", + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.6.self_attn.attn.impl.k_cache", + "language_model.model.layers.6.self_attn.attn.impl.v_cache", + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.6.feed_forward.gate_up_proj", + "language_model.model.layers.6.feed_forward.down_proj", + "language_model.model.layers.7.self_attn.qkv_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.7.self_attn.attn.impl.softmax", + "language_model.model.layers.7.self_attn.attn.impl.matmul_av", + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.7.self_attn.attn.impl.k_cache", + "language_model.model.layers.7.self_attn.attn.impl.v_cache", + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.7.feed_forward.router", + "language_model.model.layers.7.feed_forward.experts", + "language_model.model.layers.7.feed_forward.experts.moe_op", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.7.feed_forward.shared_expert.down_proj", + "language_model.model.layers.8.self_attn.qkv_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.8.self_attn.attn.impl.softmax", + "language_model.model.layers.8.self_attn.attn.impl.matmul_av", + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.8.self_attn.attn.impl.k_cache", + "language_model.model.layers.8.self_attn.attn.impl.v_cache", + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.8.feed_forward.gate_up_proj", + "language_model.model.layers.8.feed_forward.down_proj", + "language_model.model.layers.9.self_attn.qkv_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.9.self_attn.attn.impl.softmax", + "language_model.model.layers.9.self_attn.attn.impl.matmul_av", + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.9.self_attn.attn.impl.k_cache", + "language_model.model.layers.9.self_attn.attn.impl.v_cache", + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.9.feed_forward.router", + "language_model.model.layers.9.feed_forward.experts", + "language_model.model.layers.9.feed_forward.experts.moe_op", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.9.feed_forward.shared_expert.down_proj", + "language_model.model.layers.10.self_attn.qkv_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.10.self_attn.attn.impl.softmax", + "language_model.model.layers.10.self_attn.attn.impl.matmul_av", + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.10.self_attn.attn.impl.k_cache", + "language_model.model.layers.10.self_attn.attn.impl.v_cache", + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.10.feed_forward.gate_up_proj", + "language_model.model.layers.10.feed_forward.down_proj", + "language_model.model.layers.11.self_attn.qkv_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.11.self_attn.attn.impl.softmax", + "language_model.model.layers.11.self_attn.attn.impl.matmul_av", + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.11.self_attn.attn.impl.k_cache", + "language_model.model.layers.11.self_attn.attn.impl.v_cache", + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.11.feed_forward.router", + "language_model.model.layers.11.feed_forward.experts", + "language_model.model.layers.11.feed_forward.experts.moe_op", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.11.feed_forward.shared_expert.down_proj", + "language_model.model.layers.12.self_attn.qkv_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.12.self_attn.attn.impl.softmax", + "language_model.model.layers.12.self_attn.attn.impl.matmul_av", + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.12.self_attn.attn.impl.k_cache", + "language_model.model.layers.12.self_attn.attn.impl.v_cache", + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.12.feed_forward.gate_up_proj", + "language_model.model.layers.12.feed_forward.down_proj", + "language_model.model.layers.13.self_attn.qkv_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.13.self_attn.attn.impl.softmax", + "language_model.model.layers.13.self_attn.attn.impl.matmul_av", + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.13.self_attn.attn.impl.k_cache", + "language_model.model.layers.13.self_attn.attn.impl.v_cache", + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.13.feed_forward.router", + "language_model.model.layers.13.feed_forward.experts", + "language_model.model.layers.13.feed_forward.experts.moe_op", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.13.feed_forward.shared_expert.down_proj", + "language_model.model.layers.14.self_attn.qkv_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.14.self_attn.attn.impl.softmax", + "language_model.model.layers.14.self_attn.attn.impl.matmul_av", + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.14.self_attn.attn.impl.k_cache", + "language_model.model.layers.14.self_attn.attn.impl.v_cache", + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.14.feed_forward.gate_up_proj", + "language_model.model.layers.14.feed_forward.down_proj", + "language_model.model.layers.15.self_attn.qkv_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.15.self_attn.attn.impl.softmax", + "language_model.model.layers.15.self_attn.attn.impl.matmul_av", + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.15.self_attn.attn.impl.k_cache", + "language_model.model.layers.15.self_attn.attn.impl.v_cache", + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.15.feed_forward.router", + "language_model.model.layers.15.feed_forward.experts", + "language_model.model.layers.15.feed_forward.experts.moe_op", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.15.feed_forward.shared_expert.down_proj", + "language_model.model.layers.16.self_attn.qkv_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.16.self_attn.attn.impl.softmax", + "language_model.model.layers.16.self_attn.attn.impl.matmul_av", + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.16.self_attn.attn.impl.k_cache", + "language_model.model.layers.16.self_attn.attn.impl.v_cache", + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.16.feed_forward.gate_up_proj", + "language_model.model.layers.16.feed_forward.down_proj", + "language_model.model.layers.17.self_attn.qkv_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.17.self_attn.attn.impl.softmax", + "language_model.model.layers.17.self_attn.attn.impl.matmul_av", + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.17.self_attn.attn.impl.k_cache", + "language_model.model.layers.17.self_attn.attn.impl.v_cache", + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.17.feed_forward.router", + "language_model.model.layers.17.feed_forward.experts", + "language_model.model.layers.17.feed_forward.experts.moe_op", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.17.feed_forward.shared_expert.down_proj", + "language_model.model.layers.18.self_attn.qkv_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.18.self_attn.attn.impl.softmax", + "language_model.model.layers.18.self_attn.attn.impl.matmul_av", + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.18.self_attn.attn.impl.k_cache", + "language_model.model.layers.18.self_attn.attn.impl.v_cache", + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.18.feed_forward.gate_up_proj", + "language_model.model.layers.18.feed_forward.down_proj", + "language_model.model.layers.19.self_attn.qkv_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.19.self_attn.attn.impl.softmax", + "language_model.model.layers.19.self_attn.attn.impl.matmul_av", + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.19.self_attn.attn.impl.k_cache", + "language_model.model.layers.19.self_attn.attn.impl.v_cache", + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.19.feed_forward.router", + "language_model.model.layers.19.feed_forward.experts", + "language_model.model.layers.19.feed_forward.experts.moe_op", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.19.feed_forward.shared_expert.down_proj", + "language_model.model.layers.20.self_attn.qkv_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.20.self_attn.attn.impl.softmax", + "language_model.model.layers.20.self_attn.attn.impl.matmul_av", + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.20.self_attn.attn.impl.k_cache", + "language_model.model.layers.20.self_attn.attn.impl.v_cache", + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.20.feed_forward.gate_up_proj", + "language_model.model.layers.20.feed_forward.down_proj", + "language_model.model.layers.21.self_attn.qkv_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.21.self_attn.attn.impl.softmax", + "language_model.model.layers.21.self_attn.attn.impl.matmul_av", + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.21.self_attn.attn.impl.k_cache", + "language_model.model.layers.21.self_attn.attn.impl.v_cache", + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.21.feed_forward.router", + "language_model.model.layers.21.feed_forward.experts", + "language_model.model.layers.21.feed_forward.experts.moe_op", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.21.feed_forward.shared_expert.down_proj", + "language_model.model.layers.22.self_attn.qkv_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.22.self_attn.attn.impl.softmax", + "language_model.model.layers.22.self_attn.attn.impl.matmul_av", + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.22.self_attn.attn.impl.k_cache", + "language_model.model.layers.22.self_attn.attn.impl.v_cache", + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.22.feed_forward.gate_up_proj", + "language_model.model.layers.22.feed_forward.down_proj", + "language_model.model.layers.23.self_attn.qkv_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.23.self_attn.attn.impl.softmax", + "language_model.model.layers.23.self_attn.attn.impl.matmul_av", + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.23.self_attn.attn.impl.k_cache", + "language_model.model.layers.23.self_attn.attn.impl.v_cache", + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.23.feed_forward.router", + "language_model.model.layers.23.feed_forward.experts", + "language_model.model.layers.23.feed_forward.experts.moe_op", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.23.feed_forward.shared_expert.down_proj", + "language_model.model.layers.24.self_attn.qkv_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.24.self_attn.attn.impl.softmax", + "language_model.model.layers.24.self_attn.attn.impl.matmul_av", + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.24.self_attn.attn.impl.k_cache", + "language_model.model.layers.24.self_attn.attn.impl.v_cache", + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.24.feed_forward.gate_up_proj", + "language_model.model.layers.24.feed_forward.down_proj", + "language_model.model.layers.25.self_attn.qkv_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.25.self_attn.attn.impl.softmax", + "language_model.model.layers.25.self_attn.attn.impl.matmul_av", + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.25.self_attn.attn.impl.k_cache", + "language_model.model.layers.25.self_attn.attn.impl.v_cache", + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.25.feed_forward.router", + "language_model.model.layers.25.feed_forward.experts", + "language_model.model.layers.25.feed_forward.experts.moe_op", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.25.feed_forward.shared_expert.down_proj", + "language_model.model.layers.26.self_attn.qkv_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.26.self_attn.attn.impl.softmax", + "language_model.model.layers.26.self_attn.attn.impl.matmul_av", + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.26.self_attn.attn.impl.k_cache", + "language_model.model.layers.26.self_attn.attn.impl.v_cache", + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.26.feed_forward.gate_up_proj", + "language_model.model.layers.26.feed_forward.down_proj", + "language_model.model.layers.27.self_attn.qkv_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.27.self_attn.attn.impl.softmax", + "language_model.model.layers.27.self_attn.attn.impl.matmul_av", + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.27.self_attn.attn.impl.k_cache", + "language_model.model.layers.27.self_attn.attn.impl.v_cache", + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.27.feed_forward.router", + "language_model.model.layers.27.feed_forward.experts", + "language_model.model.layers.27.feed_forward.experts.moe_op", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.27.feed_forward.shared_expert.down_proj", + "language_model.model.layers.28.self_attn.qkv_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.28.self_attn.attn.impl.softmax", + "language_model.model.layers.28.self_attn.attn.impl.matmul_av", + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.28.self_attn.attn.impl.k_cache", + "language_model.model.layers.28.self_attn.attn.impl.v_cache", + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.28.feed_forward.gate_up_proj", + "language_model.model.layers.28.feed_forward.down_proj", + "language_model.model.layers.29.self_attn.qkv_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.29.self_attn.attn.impl.softmax", + "language_model.model.layers.29.self_attn.attn.impl.matmul_av", + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.29.self_attn.attn.impl.k_cache", + "language_model.model.layers.29.self_attn.attn.impl.v_cache", + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.29.feed_forward.router", + "language_model.model.layers.29.feed_forward.experts", + "language_model.model.layers.29.feed_forward.experts.moe_op", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.29.feed_forward.shared_expert.down_proj", + "language_model.model.layers.30.self_attn.qkv_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.30.self_attn.attn.impl.softmax", + "language_model.model.layers.30.self_attn.attn.impl.matmul_av", + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.30.self_attn.attn.impl.k_cache", + "language_model.model.layers.30.self_attn.attn.impl.v_cache", + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.30.feed_forward.gate_up_proj", + "language_model.model.layers.30.feed_forward.down_proj", + "language_model.model.layers.31.self_attn.qkv_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.31.self_attn.attn.impl.softmax", + "language_model.model.layers.31.self_attn.attn.impl.matmul_av", + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.31.self_attn.attn.impl.k_cache", + "language_model.model.layers.31.self_attn.attn.impl.v_cache", + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.31.feed_forward.router", + "language_model.model.layers.31.feed_forward.experts", + "language_model.model.layers.31.feed_forward.experts.moe_op", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.31.feed_forward.shared_expert.down_proj", + "language_model.model.layers.32.self_attn.qkv_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.32.self_attn.attn.impl.softmax", + "language_model.model.layers.32.self_attn.attn.impl.matmul_av", + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.32.self_attn.attn.impl.k_cache", + "language_model.model.layers.32.self_attn.attn.impl.v_cache", + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.32.feed_forward.gate_up_proj", + "language_model.model.layers.32.feed_forward.down_proj", + "language_model.model.layers.33.self_attn.qkv_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.33.self_attn.attn.impl.softmax", + "language_model.model.layers.33.self_attn.attn.impl.matmul_av", + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.33.self_attn.attn.impl.k_cache", + "language_model.model.layers.33.self_attn.attn.impl.v_cache", + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.33.feed_forward.router", + "language_model.model.layers.33.feed_forward.experts", + "language_model.model.layers.33.feed_forward.experts.moe_op", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.33.feed_forward.shared_expert.down_proj", + "language_model.model.layers.34.self_attn.qkv_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.34.self_attn.attn.impl.softmax", + "language_model.model.layers.34.self_attn.attn.impl.matmul_av", + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.34.self_attn.attn.impl.k_cache", + "language_model.model.layers.34.self_attn.attn.impl.v_cache", + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.34.feed_forward.gate_up_proj", + "language_model.model.layers.34.feed_forward.down_proj", + "language_model.model.layers.35.self_attn.qkv_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.35.self_attn.attn.impl.softmax", + "language_model.model.layers.35.self_attn.attn.impl.matmul_av", + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.35.self_attn.attn.impl.k_cache", + "language_model.model.layers.35.self_attn.attn.impl.v_cache", + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.35.feed_forward.router", + "language_model.model.layers.35.feed_forward.experts", + "language_model.model.layers.35.feed_forward.experts.moe_op", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.35.feed_forward.shared_expert.down_proj", + "language_model.model.layers.36.self_attn.qkv_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.36.self_attn.attn.impl.softmax", + "language_model.model.layers.36.self_attn.attn.impl.matmul_av", + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.36.self_attn.attn.impl.k_cache", + "language_model.model.layers.36.self_attn.attn.impl.v_cache", + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.36.feed_forward.gate_up_proj", + "language_model.model.layers.36.feed_forward.down_proj", + "language_model.model.layers.37.self_attn.qkv_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.37.self_attn.attn.impl.softmax", + "language_model.model.layers.37.self_attn.attn.impl.matmul_av", + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.37.self_attn.attn.impl.k_cache", + "language_model.model.layers.37.self_attn.attn.impl.v_cache", + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.37.feed_forward.router", + "language_model.model.layers.37.feed_forward.experts", + "language_model.model.layers.37.feed_forward.experts.moe_op", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.37.feed_forward.shared_expert.down_proj", + "language_model.model.layers.38.self_attn.qkv_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.38.self_attn.attn.impl.softmax", + "language_model.model.layers.38.self_attn.attn.impl.matmul_av", + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.38.self_attn.attn.impl.k_cache", + "language_model.model.layers.38.self_attn.attn.impl.v_cache", + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.38.feed_forward.gate_up_proj", + "language_model.model.layers.38.feed_forward.down_proj", + "language_model.model.layers.39.self_attn.qkv_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.39.self_attn.attn.impl.softmax", + "language_model.model.layers.39.self_attn.attn.impl.matmul_av", + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.39.self_attn.attn.impl.k_cache", + "language_model.model.layers.39.self_attn.attn.impl.v_cache", + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.39.feed_forward.router", + "language_model.model.layers.39.feed_forward.experts", + "language_model.model.layers.39.feed_forward.experts.moe_op", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.39.feed_forward.shared_expert.down_proj", + "language_model.model.layers.40.self_attn.qkv_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.40.self_attn.attn.impl.softmax", + "language_model.model.layers.40.self_attn.attn.impl.matmul_av", + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.40.self_attn.attn.impl.k_cache", + "language_model.model.layers.40.self_attn.attn.impl.v_cache", + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.40.feed_forward.gate_up_proj", + "language_model.model.layers.40.feed_forward.down_proj", + "language_model.model.layers.41.self_attn.qkv_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.41.self_attn.attn.impl.softmax", + "language_model.model.layers.41.self_attn.attn.impl.matmul_av", + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.41.self_attn.attn.impl.k_cache", + "language_model.model.layers.41.self_attn.attn.impl.v_cache", + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.41.feed_forward.router", + "language_model.model.layers.41.feed_forward.experts", + "language_model.model.layers.41.feed_forward.experts.moe_op", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.41.feed_forward.shared_expert.down_proj", + "language_model.model.layers.42.self_attn.qkv_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.42.self_attn.attn.impl.softmax", + "language_model.model.layers.42.self_attn.attn.impl.matmul_av", + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.42.self_attn.attn.impl.k_cache", + "language_model.model.layers.42.self_attn.attn.impl.v_cache", + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.42.feed_forward.gate_up_proj", + "language_model.model.layers.42.feed_forward.down_proj", + "language_model.model.layers.43.self_attn.qkv_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.43.self_attn.attn.impl.softmax", + "language_model.model.layers.43.self_attn.attn.impl.matmul_av", + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.43.self_attn.attn.impl.k_cache", + "language_model.model.layers.43.self_attn.attn.impl.v_cache", + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.43.feed_forward.router", + "language_model.model.layers.43.feed_forward.experts", + "language_model.model.layers.43.feed_forward.experts.moe_op", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.43.feed_forward.shared_expert.down_proj", + "language_model.model.layers.44.self_attn.qkv_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.44.self_attn.attn.impl.softmax", + "language_model.model.layers.44.self_attn.attn.impl.matmul_av", + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.44.self_attn.attn.impl.k_cache", + "language_model.model.layers.44.self_attn.attn.impl.v_cache", + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.44.feed_forward.gate_up_proj", + "language_model.model.layers.44.feed_forward.down_proj", + "language_model.model.layers.45.self_attn.qkv_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.45.self_attn.attn.impl.softmax", + "language_model.model.layers.45.self_attn.attn.impl.matmul_av", + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.45.self_attn.attn.impl.k_cache", + "language_model.model.layers.45.self_attn.attn.impl.v_cache", + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.45.feed_forward.router", + "language_model.model.layers.45.feed_forward.experts", + "language_model.model.layers.45.feed_forward.experts.moe_op", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.45.feed_forward.shared_expert.down_proj", + "language_model.model.layers.46.self_attn.qkv_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.46.self_attn.attn.impl.softmax", + "language_model.model.layers.46.self_attn.attn.impl.matmul_av", + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.46.self_attn.attn.impl.k_cache", + "language_model.model.layers.46.self_attn.attn.impl.v_cache", + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.46.feed_forward.gate_up_proj", + "language_model.model.layers.46.feed_forward.down_proj", + "language_model.model.layers.47.self_attn.qkv_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.47.self_attn.attn.impl.softmax", + "language_model.model.layers.47.self_attn.attn.impl.matmul_av", + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.47.self_attn.attn.impl.k_cache", + "language_model.model.layers.47.self_attn.attn.impl.v_cache", + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.47.feed_forward.router", + "language_model.model.layers.47.feed_forward.experts", + "language_model.model.layers.47.feed_forward.experts.moe_op", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.47.feed_forward.shared_expert.down_proj", + "language_model.lm_head" +] \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_3_8.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_3_8.json new file mode 100644 index 000000000000..97d0c2ff9312 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_3_8.json @@ -0,0 +1 @@ +{"GlobalRank": null, "LocalRank": 3, "Mode": "DynamicRange", "Nodes": {"language_model.model.layers.0.self_attn.qkv_proj": {"inputs": [[[11.1875]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.0.self_attn.o_proj": {"inputs": [[[0.7578125]]], "outputs": [[[0.80078125]], [[3.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.0.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.3984375]], [[5.25]]]}, "language_model.model.layers.0.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.890625]]]}, "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.3984375]]]}, "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.474609375]]]}, "language_model.model.layers.0.self_attn.attn.impl.k_cache": {"inputs": [[[5.25]]]}, "language_model.model.layers.0.self_attn.attn.impl.v_cache": {"inputs": [[[0.890625]]]}, "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[5.03125]], [[5.25]], [[0.890625]]], "outputs": [[[0.7578125]], [[1.0]]]}, "language_model.model.layers.0.feed_forward.gate_up_proj": {"inputs": [[[2.546875]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.0.feed_forward.down_proj": {"inputs": [[[22.375]]], "outputs": [[[26.75]], [[65.0]]], "params": {"weight": [[0.765625]]}}, "language_model.model.layers.1.self_attn.qkv_proj": {"inputs": [[[12.3125]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.1.self_attn.o_proj": {"inputs": [[[1.328125]]], "outputs": [[[0.52734375]], [[1.0078125]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.1.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8046875]], [[9.9375]]]}, "language_model.model.layers.1.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.6171875]]]}, "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8046875]]]}, "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.421875]]]}, "language_model.model.layers.1.self_attn.attn.impl.k_cache": {"inputs": [[[9.9375]]]}, "language_model.model.layers.1.self_attn.attn.impl.v_cache": {"inputs": [[[1.6171875]]]}, "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.375]], [[9.3125]], [[1.6171875]]], "outputs": [[[1.328125]], [[1.0]]]}, "language_model.model.layers.1.feed_forward.router": {"inputs": [[[1.8828125]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op": {"inputs": [[[1.8828125]]], "outputs": [[[1.046875]], [[16.375]], [[0.0]], [[0.28515625]], [[13.125]], [[10.625]], [[12.375]], [[14.5]], [[7.71875]], [[11.875]], [[9.0]], [[10.75]], [[11.9375]], [[1.5625]], [[10.8125]], [[12.125]], [[15.75]], [[3.4375]], [[5.6875]], [[11.9375]], [[5.5625]], [[12.8125]], [[11.75]], [[0.0]], [[16.375]], [[7.4375]], [[0.7265625]], [[12.875]], [[9.3125]], [[7.5]], [[13.0625]], [[18.5]], [[11.375]], [[0.365234375]], [[1.625]], [[0.0]], [[10.875]], [[1.8125]], [[11.125]], [[4.5625]], [[12.125]], [[11.8125]], [[17.125]], [[15.125]], [[11.25]], [[3.515625]], [[12.9375]], [[10.5]], [[4.90625]], [[4.3125]], [[0.0]], [[10.125]], [[11.3125]], [[8.75]], [[4.4375]], [[22.75]], [[15.0]], [[10.25]], [[0.396484375]], [[8.5]], [[13.75]], [[10.5]], [[13.1875]], [[12.375]], [[0.0]], [[16.0]], [[13.1875]], [[4.09375]], [[10.75]], [[3.203125]], [[4.1875]], [[12.375]], [[15.0]], [[13.625]], [[15.875]], [[14.4375]], [[3.265625]], [[13.5625]], [[11.6875]], [[4.53125]], [[12.3125]], [[0.0]], [[1.7734375]], [[14.125]], [[4.09375]], [[16.875]], [[10.5625]], [[10.75]], [[9.0]], [[8.8125]], [[10.8125]], [[10.6875]], [[9.9375]], [[12.0]], [[0.0]], [[12.75]], [[11.0625]], [[0.0732421875]], [[0.0]], [[12.0625]], [[15.375]], [[10.0]], [[1.78125]], [[9.3125]], [[4.8125]], [[0.8515625]], [[6.71875]], [[0.07763671875]], [[0.58984375]], [[4.71875]], [[0.2041015625]], [[11.625]], [[1.453125]], [[14.5]], [[0.0]], [[12.875]], [[13.25]], [[13.1875]], [[11.5]], [[14.1875]], [[9.6875]], [[11.0625]], [[13.1875]], [[5.21875]], [[4.96875]], [[11.375]], [[7.1875]], [[13.875]], [[12.6875]]]}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.431640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.36328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.07470703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.07470703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.357421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.392578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[1.8828125]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.1.feed_forward.shared_expert.down_proj": {"inputs": [[[5.03125]]], "outputs": [[[0.310546875]], [[2.007134273635223e+38]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.2.self_attn.qkv_proj": {"inputs": [[[9.875]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.2.self_attn.o_proj": {"inputs": [[[0.8203125]]], "outputs": [[[0.6875]], [[1.328125]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.2.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.5703125]], [[8.9375]]]}, "language_model.model.layers.2.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.0]]]}, "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.5703125]]]}, "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.7578125]]]}, "language_model.model.layers.2.self_attn.attn.impl.k_cache": {"inputs": [[[8.9375]]]}, "language_model.model.layers.2.self_attn.attn.impl.v_cache": {"inputs": [[[1.0]]]}, "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.5625]], [[8.9375]], [[1.0]]], "outputs": [[[0.8203125]], [[1.0]]]}, "language_model.model.layers.2.feed_forward.gate_up_proj": {"inputs": [[[6.125]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.2.feed_forward.down_proj": {"inputs": [[[4.4375]]], "outputs": [[[4.75]], [[25.25]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.3.self_attn.qkv_proj": {"inputs": [[[7.21875]]], "params": {"weight": [[0.41796875]]}}, "language_model.model.layers.3.self_attn.o_proj": {"inputs": [[[0.3203125]]], "outputs": [[[0.59765625]], [[2.40625]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.3.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.13671875]], [[3.421875]]]}, "language_model.model.layers.3.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.3203125]]]}, "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.13671875]]]}, "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.3203125]]]}, "language_model.model.layers.3.self_attn.attn.impl.k_cache": {"inputs": [[[3.421875]]]}, "language_model.model.layers.3.self_attn.attn.impl.v_cache": {"inputs": [[[0.3203125]]]}, "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[1.890625]], [[3.421875]], [[0.310546875]]], "outputs": [[[0.294921875]], [[1.0]]]}, "language_model.model.layers.3.feed_forward.router": {"inputs": [[[8.75]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op": {"inputs": [[[8.75]]], "outputs": [[[179.0]], [[1.8671875]], [[10.25]], [[18.875]], [[11.5625]], [[0.0]], [[2.828125]], [[16.125]], [[9.9375]], [[12.8125]], [[17.375]], [[7.15625]], [[11.125]], [[13.75]], [[17.75]], [[11.0]], [[15.125]], [[1.8984375]], [[4.65625]], [[2.453125]], [[14.4375]], [[6.4375]], [[15.9375]], [[13.8125]], [[14.4375]], [[8.4375]], [[2.609375]], [[10.0]], [[6.625]], [[7.34375]], [[12.3125]], [[13.5]], [[5.4375]], [[5.5625]], [[7.71875]], [[14.0625]], [[11.9375]], [[3.515625]], [[0.7421875]], [[13.25]], [[3.0625]], [[4.28125]], [[14.5625]], [[1.8046875]], [[9.75]], [[17.75]], [[3.0]], [[12.375]], [[15.0625]], [[3.328125]], [[13.375]], [[11.0]], [[12.25]], [[10.375]], [[1.0625]], [[8.625]], [[13.125]], [[7.3125]], [[15.25]], [[15.4375]], [[10.5625]], [[5.0625]], [[16.5]], [[11.75]], [[0.287109375]], [[14.125]], [[17.0]], [[17.5]], [[0.431640625]], [[5.625]], [[0.0]], [[14.1875]], [[8.25]], [[10.75]], [[9.375]], [[6.40625]], [[9.5]], [[14.9375]], [[0.1259765625]], [[18.5]], [[14.875]], [[15.625]], [[0.0]], [[5.09375]], [[20.0]], [[8.5625]], [[24.75]], [[5.375]], [[2.734375]], [[18.5]], [[16.875]], [[13.875]], [[3.484375]], [[7.96875]], [[13.6875]], [[0.0]], [[18.0]], [[16.5]], [[13.375]], [[13.25]], [[1.46875]], [[17.0]], [[2.8125]], [[6.375]], [[13.9375]], [[0.0]], [[2.09375]], [[12.125]], [[16.5]], [[11.4375]], [[1.2734375]], [[14.4375]], [[8.125]], [[16.5]], [[0.83203125]], [[6.78125]], [[2.171875]], [[14.0625]], [[0.5703125]], [[1.703125]], [[6.375]], [[23.875]], [[5.71875]], [[8.4375]], [[11.625]], [[3.9375]], [[15.5]], [[5.71875]], [[11.75]]]}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.07470703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.75]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.37109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[8.75]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.3.feed_forward.shared_expert.down_proj": {"inputs": [[[5.6875]]], "outputs": [[[0.474609375]], [[2.007134273635223e+38]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.4.self_attn.qkv_proj": {"inputs": [[[13.75]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.4.self_attn.o_proj": {"inputs": [[[2.015625]]], "outputs": [[[1.015625]], [[2.046875]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.4.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.796875]], [[12.0]]]}, "language_model.model.layers.4.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.8125]]]}, "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.796875]]]}, "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.234375]]]}, "language_model.model.layers.4.self_attn.attn.impl.k_cache": {"inputs": [[[12.0]]]}, "language_model.model.layers.4.self_attn.attn.impl.v_cache": {"inputs": [[[2.8125]]]}, "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.5]], [[12.0]], [[2.8125]]], "outputs": [[[2.015625]], [[1.0]]]}, "language_model.model.layers.4.feed_forward.gate_up_proj": {"inputs": [[[7.09375]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.4.feed_forward.down_proj": {"inputs": [[[9.875]]], "outputs": [[[2.796875]], [[4.96875]]], "params": {"weight": [[0.482421875]]}}, "language_model.model.layers.5.self_attn.qkv_proj": {"inputs": [[[17.375]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.5.self_attn.o_proj": {"inputs": [[[1.8515625]]], "outputs": [[[0.97265625]], [[2.21875]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.5.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.72265625]], [[16.25]]]}, "language_model.model.layers.5.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.734375]]]}, "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.72265625]]]}, "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.609375]]]}, "language_model.model.layers.5.self_attn.attn.impl.k_cache": {"inputs": [[[16.25]]]}, "language_model.model.layers.5.self_attn.attn.impl.v_cache": {"inputs": [[[3.734375]]]}, "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.625]], [[16.25]], [[3.734375]]], "outputs": [[[1.8515625]], [[1.0]]]}, "language_model.model.layers.5.feed_forward.router": {"inputs": [[[19.25]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op": {"inputs": [[[19.25]]], "outputs": [[[188.0]], [[2.40625]], [[2.25]], [[8.6875]], [[14.875]], [[11.0625]], [[8.4375]], [[4.53125]], [[17.0]], [[19.875]], [[24.375]], [[6.75]], [[16.25]], [[21.875]], [[2.96875]], [[9.4375]], [[23.5]], [[6.75]], [[9.0625]], [[15.9375]], [[4.34375]], [[23.5]], [[5.75]], [[6.53125]], [[15.9375]], [[3.953125]], [[3.015625]], [[12.375]], [[171.0]], [[8.6875]], [[20.25]], [[24.375]], [[5.5]], [[7.0625]], [[27.5]], [[10.0625]], [[25.625]], [[13.5]], [[3.78125]], [[14.875]], [[11.0625]], [[11.1875]], [[2.828125]], [[11.25]], [[23.5]], [[12.4375]], [[12.6875]], [[10.1875]], [[16.0]], [[4.59375]], [[19.875]], [[7.21875]], [[13.0625]], [[2.21875]], [[3.03125]], [[16.0]], [[2.8125]], [[7.71875]], [[19.75]], [[14.5625]], [[0.76953125]], [[11.375]], [[14.3125]], [[9.625]], [[17.375]], [[3.375]], [[9.5]], [[8.5625]], [[8.1875]], [[16.375]], [[8.25]], [[7.5625]], [[17.625]], [[25.25]], [[12.875]], [[3.6875]], [[22.125]], [[23.125]], [[10.125]], [[16.75]], [[14.25]], [[1.34375]], [[17.875]], [[12.1875]], [[18.375]], [[4.40625]], [[15.3125]], [[0.2734375]], [[10.9375]], [[16.375]], [[26.375]], [[19.0]], [[8.75]], [[10.5625]], [[12.5]], [[12.0]], [[6.4375]], [[17.25]], [[9.625]], [[9.375]], [[21.0]], [[11.125]], [[5.25]], [[4.34375]], [[23.25]], [[20.375]], [[18.5]], [[4.25]], [[26.0]], [[11.625]], [[15.75]], [[10.625]], [[4.8125]], [[6.25]], [[4.96875]], [[18.125]], [[23.875]], [[2.65625]], [[6.1875]], [[1.875]], [[15.125]], [[15.125]], [[5.21875]], [[4.34375]], [[16.375]], [[16.375]], [[1.640625]], [[10.3125]], [[2.75]]]}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.3984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.78125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.6640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.58984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.6796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[19.25]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.5.feed_forward.shared_expert.down_proj": {"inputs": [[[6.125]]], "outputs": [[[0.55078125]], [[2.007134273635223e+38]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.6.self_attn.qkv_proj": {"inputs": [[[12.9375]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.6.self_attn.o_proj": {"inputs": [[[1.546875]]], "outputs": [[[0.66796875]], [[1.5234375]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.6.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.76953125]], [[14.75]]]}, "language_model.model.layers.6.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.5]]]}, "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.76953125]]]}, "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.515625]]]}, "language_model.model.layers.6.self_attn.attn.impl.k_cache": {"inputs": [[[14.75]]]}, "language_model.model.layers.6.self_attn.attn.impl.v_cache": {"inputs": [[[2.5]]]}, "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.5625]], [[14.75]], [[2.5]]], "outputs": [[[1.546875]], [[1.0]]]}, "language_model.model.layers.6.feed_forward.gate_up_proj": {"inputs": [[[7.71875]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.6.feed_forward.down_proj": {"inputs": [[[7.21875]]], "outputs": [[[2.328125]], [[11.1875]]], "params": {"weight": [[0.609375]]}}, "language_model.model.layers.7.self_attn.qkv_proj": {"inputs": [[[7.09375]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.7.self_attn.o_proj": {"inputs": [[[1.140625]]], "outputs": [[[1.84375]], [[3.84375]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.7.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.408203125]], [[4.9375]]]}, "language_model.model.layers.7.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.5703125]]]}, "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.408203125]]]}, "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.25]]]}, "language_model.model.layers.7.self_attn.attn.impl.k_cache": {"inputs": [[[4.9375]]]}, "language_model.model.layers.7.self_attn.attn.impl.v_cache": {"inputs": [[[1.5703125]]]}, "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[4.8125]], [[4.6875]], [[1.5703125]]], "outputs": [[[1.140625]], [[1.0]]]}, "language_model.model.layers.7.feed_forward.router": {"inputs": [[[11.6875]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op": {"inputs": [[[11.6875]]], "outputs": [[[2.875]], [[7.28125]], [[9.0625]], [[13.1875]], [[12.5]], [[7.9375]], [[14.3125]], [[8.6875]], [[21.125]], [[12.9375]], [[13.1875]], [[17.0]], [[16.375]], [[21.875]], [[20.625]], [[7.46875]], [[1.828125]], [[13.0]], [[9.375]], [[23.625]], [[13.9375]], [[15.375]], [[24.375]], [[19.125]], [[31.875]], [[16.125]], [[11.5625]], [[4.40625]], [[5.3125]], [[16.125]], [[6.625]], [[15.1875]], [[46.25]], [[9.0625]], [[4.9375]], [[2.9375]], [[9.8125]], [[4.75]], [[16.125]], [[28.875]], [[2.328125]], [[1.7578125]], [[6.71875]], [[1.1640625]], [[26.875]], [[13.0]], [[20.75]], [[0.53515625]], [[22.25]], [[18.375]], [[12.0625]], [[14.6875]], [[26.875]], [[16.0]], [[10.875]], [[22.125]], [[29.5]], [[19.25]], [[2.65625]], [[3.875]], [[3.015625]], [[8.375]], [[17.875]], [[3.734375]], [[5.75]], [[21.875]], [[17.125]], [[15.8125]], [[10.3125]], [[9.8125]], [[4.0625]], [[24.5]], [[4.9375]], [[11.3125]], [[12.25]], [[27.25]], [[5.03125]], [[4.46875]], [[7.46875]], [[0.86328125]], [[5.34375]], [[18.25]], [[15.8125]], [[3.90625]], [[7.21875]], [[2.421875]], [[13.0]], [[1.625]], [[19.875]], [[6.375]], [[25.75]], [[11.625]], [[12.9375]], [[17.25]], [[3.859375]], [[24.875]], [[6.15625]], [[18.75]], [[5.5625]], [[35.5]], [[3.03125]], [[10.75]], [[15.375]], [[4.375]], [[1.015625]], [[24.5]], [[30.25]], [[10.875]], [[9.9375]], [[1.28125]], [[22.5]], [[15.4375]], [[19.0]], [[21.625]], [[31.625]], [[25.25]], [[1.9921875]], [[8.5]], [[14.25]], [[1.875]], [[19.0]], [[33.5]], [[24.25]], [[30.625]], [[12.625]], [[5.625]], [[3.03125]], [[30.375]], [[9.0625]]]}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.07373046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.07470703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.53515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.61328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.61328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.62890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[11.6875]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.7.feed_forward.shared_expert.down_proj": {"inputs": [[[11.625]]], "outputs": [[[0.9453125]], [[2.007134273635223e+38]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.8.self_attn.qkv_proj": {"inputs": [[[15.125]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.8.self_attn.o_proj": {"inputs": [[[2.359375]]], "outputs": [[[1.9375]], [[2.109375]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.8.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.82421875]], [[15.9375]]]}, "language_model.model.layers.8.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.546875]]]}, "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.82421875]]]}, "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.40625]]]}, "language_model.model.layers.8.self_attn.attn.impl.k_cache": {"inputs": [[[15.9375]]]}, "language_model.model.layers.8.self_attn.attn.impl.v_cache": {"inputs": [[[3.546875]]]}, "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.625]], [[15.9375]], [[3.546875]]], "outputs": [[[2.359375]], [[1.0]]]}, "language_model.model.layers.8.feed_forward.gate_up_proj": {"inputs": [[[8.25]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.8.feed_forward.down_proj": {"inputs": [[[8.6875]]], "outputs": [[[2.5625]], [[9.25]]], "params": {"weight": [[0.59375]]}}, "language_model.model.layers.9.self_attn.qkv_proj": {"inputs": [[[15.75]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.9.self_attn.o_proj": {"inputs": [[[2.390625]]], "outputs": [[[1.3125]], [[2.875]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.9.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.90625]], [[17.125]]]}, "language_model.model.layers.9.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.234375]]]}, "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.90625]]]}, "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.296875]]]}, "language_model.model.layers.9.self_attn.attn.impl.k_cache": {"inputs": [[[17.125]]]}, "language_model.model.layers.9.self_attn.attn.impl.v_cache": {"inputs": [[[3.234375]]]}, "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.9375]], [[17.125]], [[3.234375]]], "outputs": [[[2.390625]], [[1.0]]]}, "language_model.model.layers.9.feed_forward.router": {"inputs": [[[11.75]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op": {"inputs": [[[11.75]]], "outputs": [[[3.859375]], [[10.875]], [[11.625]], [[2.09375]], [[32.75]], [[26.375]], [[32.0]], [[30.5]], [[67.0]], [[14.375]], [[28.625]], [[17.875]], [[7.9375]], [[36.5]], [[47.25]], [[18.75]], [[5.40625]], [[24.25]], [[35.75]], [[46.25]], [[14.8125]], [[13.8125]], [[16.0]], [[18.375]], [[19.625]], [[10.125]], [[32.75]], [[43.25]], [[47.5]], [[14.1875]], [[38.25]], [[14.25]], [[16.125]], [[38.5]], [[17.375]], [[52.5]], [[15.9375]], [[54.5]], [[22.375]], [[7.15625]], [[18.75]], [[6.1875]], [[8.6875]], [[15.0]], [[26.125]], [[7.875]], [[12.6875]], [[29.375]], [[17.125]], [[26.5]], [[33.75]], [[13.125]], [[28.5]], [[16.25]], [[10.625]], [[17.875]], [[9.125]], [[32.25]], [[15.125]], [[38.25]], [[22.875]], [[28.125]], [[14.5]], [[23.125]], [[41.75]], [[14.75]], [[25.0]], [[33.75]], [[40.25]], [[8.1875]], [[16.25]], [[26.25]], [[49.75]], [[11.25]], [[10.875]], [[4.03125]], [[27.25]], [[10.375]], [[50.5]], [[3.390625]], [[28.75]], [[21.375]], [[27.75]], [[24.25]], [[28.75]], [[12.9375]], [[31.875]], [[10.75]], [[14.0625]], [[26.875]], [[32.75]], [[8.875]], [[11.875]], [[20.75]], [[9.75]], [[20.0]], [[27.875]], [[15.9375]], [[9.9375]], [[9.125]], [[14.4375]], [[26.25]], [[14.625]], [[42.5]], [[10.8125]], [[21.75]], [[13.3125]], [[23.5]], [[22.125]], [[7.625]], [[28.75]], [[21.625]], [[4.34375]], [[27.875]], [[7.09375]], [[27.375]], [[26.0]], [[12.875]], [[6.0625]], [[8.6875]], [[5.65625]], [[16.625]], [[8.6875]], [[36.75]], [[19.375]], [[38.5]], [[20.0]], [[11.125]], [[17.75]]]}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.53125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[11.75]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.9.feed_forward.shared_expert.down_proj": {"inputs": [[[12.3125]]], "outputs": [[[2.734375]], [[2.007134273635223e+38]]], "params": {"weight": [[0.4375]]}}, "language_model.model.layers.10.self_attn.qkv_proj": {"inputs": [[[16.125]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.10.self_attn.o_proj": {"inputs": [[[1.5625]]], "outputs": [[[1.6015625]], [[3.09375]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.10.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8828125]], [[17.875]]]}, "language_model.model.layers.10.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.453125]]]}, "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8828125]]]}, "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.8671875]]]}, "language_model.model.layers.10.self_attn.attn.impl.k_cache": {"inputs": [[[17.875]]]}, "language_model.model.layers.10.self_attn.attn.impl.v_cache": {"inputs": [[[3.453125]]]}, "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.6875]], [[17.875]], [[3.453125]]], "outputs": [[[1.5625]], [[1.0]]]}, "language_model.model.layers.10.feed_forward.gate_up_proj": {"inputs": [[[9.875]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.10.feed_forward.down_proj": {"inputs": [[[11.0]]], "outputs": [[[6.90625]], [[11.3125]]], "params": {"weight": [[0.515625]]}}, "language_model.model.layers.11.self_attn.qkv_proj": {"inputs": [[[13.5625]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.11.self_attn.o_proj": {"inputs": [[[2.875]]], "outputs": [[[8.0625]], [[7.875]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.11.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.81640625]], [[9.5]]]}, "language_model.model.layers.11.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.4375]]]}, "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.81640625]]]}, "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.046875]]]}, "language_model.model.layers.11.self_attn.attn.impl.k_cache": {"inputs": [[[9.5]]]}, "language_model.model.layers.11.self_attn.attn.impl.v_cache": {"inputs": [[[3.4375]]]}, "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.5625]], [[9.5]], [[3.4375]]], "outputs": [[[2.875]], [[1.0]]]}, "language_model.model.layers.11.feed_forward.router": {"inputs": [[[13.375]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op": {"inputs": [[[13.375]]], "outputs": [[[3.734375]], [[24.375]], [[14.5]], [[31.875]], [[6.9375]], [[34.75]], [[21.0]], [[1.328125]], [[21.5]], [[16.375]], [[10.25]], [[35.0]], [[23.0]], [[35.5]], [[47.25]], [[30.75]], [[58.5]], [[35.0]], [[44.75]], [[39.25]], [[21.125]], [[16.875]], [[27.125]], [[26.75]], [[17.875]], [[23.375]], [[44.0]], [[32.0]], [[10.8125]], [[7.09375]], [[39.5]], [[40.75]], [[9.0]], [[9.25]], [[16.625]], [[24.875]], [[20.375]], [[11.125]], [[16.25]], [[14.5]], [[28.0]], [[4.21875]], [[17.0]], [[9.6875]], [[40.75]], [[9.375]], [[3.78125]], [[16.5]], [[41.0]], [[31.75]], [[18.375]], [[40.75]], [[26.375]], [[10.3125]], [[23.625]], [[64.0]], [[33.25]], [[22.875]], [[10.75]], [[56.25]], [[25.875]], [[56.5]], [[20.25]], [[29.75]], [[20.625]], [[13.4375]], [[20.0]], [[43.5]], [[45.0]], [[26.5]], [[26.75]], [[4.65625]], [[7.25]], [[1.3359375]], [[30.0]], [[20.875]], [[19.25]], [[42.0]], [[4.875]], [[59.5]], [[35.5]], [[23.0]], [[11.5625]], [[19.375]], [[9.75]], [[4.25]], [[3.546875]], [[34.5]], [[10.3125]], [[17.5]], [[24.625]], [[49.75]], [[12.0]], [[59.5]], [[39.75]], [[12.0]], [[12.5625]], [[27.75]], [[17.25]], [[9.75]], [[26.125]], [[8.8125]], [[49.75]], [[41.75]], [[17.75]], [[45.25]], [[18.375]], [[14.375]], [[11.125]], [[17.5]], [[9.5625]], [[15.0]], [[10.5]], [[1.9375]], [[15.5625]], [[32.5]], [[25.125]], [[26.75]], [[46.5]], [[13.375]], [[23.875]], [[28.5]], [[13.1875]], [[12.0]], [[21.25]], [[9.375]], [[9.3125]], [[38.5]], [[12.9375]]]}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.35546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.408203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[13.375]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.11.feed_forward.shared_expert.down_proj": {"inputs": [[[31.25]]], "outputs": [[[2.0625]], [[2.007134273635223e+38]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.12.self_attn.qkv_proj": {"inputs": [[[18.625]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.12.self_attn.o_proj": {"inputs": [[[3.265625]]], "outputs": [[[1.3359375]], [[3.640625]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.12.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.97265625]], [[17.0]]]}, "language_model.model.layers.12.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.875]]]}, "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.97265625]]]}, "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.953125]]]}, "language_model.model.layers.12.self_attn.attn.impl.k_cache": {"inputs": [[[17.0]]]}, "language_model.model.layers.12.self_attn.attn.impl.v_cache": {"inputs": [[[3.875]]]}, "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.0625]], [[17.0]], [[3.875]]], "outputs": [[[3.265625]], [[1.0]]]}, "language_model.model.layers.12.feed_forward.gate_up_proj": {"inputs": [[[10.75]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.12.feed_forward.down_proj": {"inputs": [[[13.75]]], "outputs": [[[9.0]], [[17.375]]], "params": {"weight": [[0.6484375]]}}, "language_model.model.layers.13.self_attn.qkv_proj": {"inputs": [[[20.875]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.13.self_attn.o_proj": {"inputs": [[[2.53125]]], "outputs": [[[1.6875]], [[6.9375]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.13.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.81640625]], [[15.1875]]]}, "language_model.model.layers.13.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.515625]]]}, "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.81640625]]]}, "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.734375]]]}, "language_model.model.layers.13.self_attn.attn.impl.k_cache": {"inputs": [[[15.1875]]]}, "language_model.model.layers.13.self_attn.attn.impl.v_cache": {"inputs": [[[3.515625]]]}, "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.25]], [[15.1875]], [[3.515625]]], "outputs": [[[2.53125]], [[1.0]]]}, "language_model.model.layers.13.feed_forward.router": {"inputs": [[[17.625]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op": {"inputs": [[[17.625]]], "outputs": [[[4.5625]], [[22.0]], [[28.625]], [[18.375]], [[58.0]], [[46.0]], [[14.875]], [[14.4375]], [[10.125]], [[18.25]], [[2.328125]], [[39.75]], [[29.75]], [[24.0]], [[19.375]], [[26.875]], [[18.625]], [[20.75]], [[21.25]], [[10.625]], [[40.25]], [[27.125]], [[20.625]], [[6.71875]], [[11.375]], [[21.875]], [[35.25]], [[13.3125]], [[43.25]], [[36.25]], [[9.375]], [[10.3125]], [[18.25]], [[35.5]], [[19.5]], [[27.5]], [[26.375]], [[55.25]], [[42.0]], [[14.75]], [[6.03125]], [[19.625]], [[55.0]], [[32.0]], [[17.125]], [[21.5]], [[39.25]], [[25.375]], [[38.75]], [[7.0625]], [[44.0]], [[34.0]], [[41.0]], [[7.9375]], [[59.25]], [[5.40625]], [[43.5]], [[29.5]], [[46.5]], [[27.375]], [[40.75]], [[6.375]], [[30.625]], [[12.1875]], [[37.75]], [[9.6875]], [[54.0]], [[3.59375]], [[38.5]], [[37.75]], [[12.1875]], [[74.0]], [[37.25]], [[9.625]], [[6.875]], [[22.875]], [[21.25]], [[54.5]], [[29.375]], [[37.75]], [[21.0]], [[11.8125]], [[39.5]], [[2.28125]], [[41.75]], [[44.0]], [[40.0]], [[19.625]], [[23.375]], [[2.703125]], [[52.0]], [[25.5]], [[50.0]], [[3.34375]], [[55.75]], [[34.75]], [[16.5]], [[8.75]], [[11.125]], [[15.3125]], [[65.0]], [[9.625]], [[2.140625]], [[20.0]], [[29.75]], [[44.0]], [[36.25]], [[56.75]], [[43.5]], [[16.625]], [[43.0]], [[29.25]], [[26.875]], [[8.1875]], [[26.25]], [[37.0]], [[35.75]], [[13.5]], [[37.5]], [[30.25]], [[23.625]], [[39.25]], [[27.25]], [[25.875]], [[11.9375]], [[14.1875]], [[28.0]], [[15.25]], [[27.75]]]}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.3828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.474609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[17.625]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.13.feed_forward.shared_expert.down_proj": {"inputs": [[[13.25]]], "outputs": [[[2.671875]], [[2.007134273635223e+38]]], "params": {"weight": [[0.55078125]]}}, "language_model.model.layers.14.self_attn.qkv_proj": {"inputs": [[[18.75]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.14.self_attn.o_proj": {"inputs": [[[2.734375]]], "outputs": [[[2.140625]], [[4.875]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.14.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8828125]], [[15.9375]]]}, "language_model.model.layers.14.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.078125]]]}, "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8828125]]]}, "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.359375]]]}, "language_model.model.layers.14.self_attn.attn.impl.k_cache": {"inputs": [[[15.9375]]]}, "language_model.model.layers.14.self_attn.attn.impl.v_cache": {"inputs": [[[3.078125]]]}, "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.5625]], [[15.9375]], [[3.078125]]], "outputs": [[[2.734375]], [[1.0]]]}, "language_model.model.layers.14.feed_forward.gate_up_proj": {"inputs": [[[12.0625]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.14.feed_forward.down_proj": {"inputs": [[[15.5]]], "outputs": [[[4.65625]], [[28.25]]], "params": {"weight": [[0.64453125]]}}, "language_model.model.layers.15.self_attn.qkv_proj": {"inputs": [[[20.875]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.15.self_attn.o_proj": {"inputs": [[[4.125]]], "outputs": [[[21.875]], [[23.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.15.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.6875]], [[11.0]]]}, "language_model.model.layers.15.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.75]]]}, "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.6875]]]}, "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.1875]]]}, "language_model.model.layers.15.self_attn.attn.impl.k_cache": {"inputs": [[[11.0]]]}, "language_model.model.layers.15.self_attn.attn.impl.v_cache": {"inputs": [[[4.75]]]}, "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.125]], [[11.0]], [[4.75]]], "outputs": [[[4.125]], [[1.0]]]}, "language_model.model.layers.15.feed_forward.router": {"inputs": [[[19.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op": {"inputs": [[[19.0]]], "outputs": [[[4.3125]], [[6.4375]], [[10.8125]], [[42.5]], [[18.625]], [[16.5]], [[45.5]], [[44.5]], [[37.0]], [[75.5]], [[35.0]], [[18.375]], [[13.625]], [[42.75]], [[17.125]], [[4.375]], [[38.25]], [[26.625]], [[44.25]], [[13.375]], [[9.375]], [[34.25]], [[25.75]], [[39.75]], [[32.25]], [[10.5625]], [[12.375]], [[39.0]], [[18.375]], [[51.75]], [[17.75]], [[41.5]], [[30.25]], [[16.875]], [[11.75]], [[33.5]], [[22.0]], [[7.8125]], [[44.0]], [[54.25]], [[52.75]], [[39.75]], [[38.75]], [[53.5]], [[36.75]], [[41.5]], [[27.5]], [[11.0]], [[28.125]], [[18.75]], [[29.0]], [[31.0]], [[55.5]], [[42.75]], [[17.375]], [[38.5]], [[38.75]], [[21.25]], [[25.75]], [[20.375]], [[36.5]], [[14.125]], [[66.5]], [[31.0]], [[61.0]], [[15.3125]], [[66.0]], [[19.0]], [[42.25]], [[8.0]], [[32.0]], [[35.75]], [[17.5]], [[25.0]], [[62.75]], [[48.5]], [[58.5]], [[31.5]], [[61.25]], [[26.625]], [[7.34375]], [[49.75]], [[41.0]], [[28.0]], [[14.5]], [[15.4375]], [[45.0]], [[48.75]], [[19.375]], [[33.0]], [[26.0]], [[7.53125]], [[16.375]], [[22.5]], [[24.625]], [[38.5]], [[32.5]], [[22.5]], [[40.75]], [[11.5]], [[31.5]], [[44.25]], [[49.25]], [[45.75]], [[36.0]], [[51.75]], [[34.5]], [[20.25]], [[29.875]], [[60.0]], [[37.25]], [[43.75]], [[37.0]], [[57.0]], [[2.953125]], [[17.125]], [[14.25]], [[55.75]], [[14.3125]], [[54.0]], [[28.75]], [[61.5]], [[34.25]], [[28.25]], [[33.0]], [[53.25]], [[8.9375]], [[13.5]], [[45.0]]]}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[19.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.15.feed_forward.shared_expert.down_proj": {"inputs": [[[16.5]]], "outputs": [[[7.25]], [[2.007134273635223e+38]]], "params": {"weight": [[0.494140625]]}}, "language_model.model.layers.16.self_attn.qkv_proj": {"inputs": [[[20.375]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.16.self_attn.o_proj": {"inputs": [[[4.1875]]], "outputs": [[[2.75]], [[6.28125]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.16.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8515625]], [[15.0625]]]}, "language_model.model.layers.16.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.8125]]]}, "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8515625]]]}, "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.1875]]]}, "language_model.model.layers.16.self_attn.attn.impl.k_cache": {"inputs": [[[15.0625]]]}, "language_model.model.layers.16.self_attn.attn.impl.v_cache": {"inputs": [[[5.8125]]]}, "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.6875]], [[15.0625]], [[5.8125]]], "outputs": [[[4.1875]], [[1.0]]]}, "language_model.model.layers.16.feed_forward.gate_up_proj": {"inputs": [[[14.8125]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.16.feed_forward.down_proj": {"inputs": [[[19.0]]], "outputs": [[[7.03125]], [[22.625]]], "params": {"weight": [[0.6640625]]}}, "language_model.model.layers.17.self_attn.qkv_proj": {"inputs": [[[19.875]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.17.self_attn.o_proj": {"inputs": [[[4.0]]], "outputs": [[[17.5]], [[20.75]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.17.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.67578125]], [[10.6875]]]}, "language_model.model.layers.17.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.15625]]]}, "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.67578125]]]}, "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.875]]]}, "language_model.model.layers.17.self_attn.attn.impl.k_cache": {"inputs": [[[10.6875]]]}, "language_model.model.layers.17.self_attn.attn.impl.v_cache": {"inputs": [[[6.15625]]]}, "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.125]], [[10.6875]], [[6.15625]]], "outputs": [[[4.0]], [[1.0]]]}, "language_model.model.layers.17.feed_forward.router": {"inputs": [[[23.875]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op": {"inputs": [[[23.875]]], "outputs": [[[5.5625]], [[52.25]], [[27.625]], [[33.5]], [[14.375]], [[46.0]], [[32.75]], [[74.5]], [[27.875]], [[52.75]], [[4.21875]], [[12.0625]], [[12.5625]], [[22.5]], [[27.5]], [[30.625]], [[37.75]], [[32.5]], [[42.75]], [[28.0]], [[17.75]], [[32.25]], [[28.375]], [[54.0]], [[44.75]], [[12.0]], [[30.75]], [[10.375]], [[35.5]], [[34.75]], [[37.75]], [[39.0]], [[21.375]], [[13.125]], [[14.1875]], [[27.75]], [[51.25]], [[27.75]], [[38.25]], [[13.875]], [[53.75]], [[17.375]], [[52.0]], [[13.5625]], [[20.5]], [[30.125]], [[51.25]], [[47.5]], [[71.5]], [[34.5]], [[35.5]], [[73.0]], [[55.75]], [[9.5]], [[22.25]], [[13.4375]], [[7.0625]], [[6.8125]], [[42.0]], [[16.5]], [[24.125]], [[10.9375]], [[26.125]], [[9.1875]], [[15.9375]], [[56.5]], [[67.0]], [[13.5]], [[57.0]], [[45.5]], [[27.625]], [[31.125]], [[23.625]], [[43.25]], [[30.625]], [[18.875]], [[17.125]], [[39.75]], [[56.25]], [[10.0625]], [[61.75]], [[18.875]], [[70.0]], [[30.625]], [[74.0]], [[59.25]], [[44.25]], [[28.625]], [[21.0]], [[44.25]], [[47.75]], [[38.5]], [[43.5]], [[9.1875]], [[46.0]], [[49.5]], [[33.5]], [[9.5625]], [[4.59375]], [[30.75]], [[21.625]], [[6.15625]], [[11.0625]], [[34.0]], [[45.75]], [[39.75]], [[22.25]], [[14.0625]], [[38.25]], [[30.125]], [[9.8125]], [[42.75]], [[37.0]], [[54.0]], [[43.0]], [[23.5]], [[12.0]], [[52.25]], [[11.25]], [[26.25]], [[37.5]], [[48.5]], [[19.625]], [[37.0]], [[13.375]], [[32.75]], [[8.8125]], [[15.0]], [[47.0]]]}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.46875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.875]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.17.feed_forward.shared_expert.down_proj": {"inputs": [[[50.25]]], "outputs": [[[5.5]], [[2.007134273635223e+38]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.18.self_attn.qkv_proj": {"inputs": [[[22.125]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.18.self_attn.o_proj": {"inputs": [[[4.28125]]], "outputs": [[[6.375]], [[11.5625]]], "params": {"weight": [[0.59765625]]}}, "language_model.model.layers.18.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.81640625]], [[8.8125]]]}, "language_model.model.layers.18.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.1875]]]}, "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.81640625]]]}, "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.515625]]]}, "language_model.model.layers.18.self_attn.attn.impl.k_cache": {"inputs": [[[8.8125]]]}, "language_model.model.layers.18.self_attn.attn.impl.v_cache": {"inputs": [[[5.1875]]]}, "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.625]], [[8.8125]], [[5.1875]]], "outputs": [[[4.28125]], [[1.0]]]}, "language_model.model.layers.18.feed_forward.gate_up_proj": {"inputs": [[[18.5]]], "params": {"weight": [[0.435546875]]}}, "language_model.model.layers.18.feed_forward.down_proj": {"inputs": [[[16.75]]], "outputs": [[[8.25]], [[358.0]]], "params": {"weight": [[0.6953125]]}}, "language_model.model.layers.19.self_attn.qkv_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.19.self_attn.o_proj": {"inputs": [[[3.015625]]], "outputs": [[[31.625]], [[36.75]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.19.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.7890625]], [[8.8125]]]}, "language_model.model.layers.19.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.3125]]]}, "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.7890625]]]}, "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.234375]]]}, "language_model.model.layers.19.self_attn.attn.impl.k_cache": {"inputs": [[[8.8125]]]}, "language_model.model.layers.19.self_attn.attn.impl.v_cache": {"inputs": [[[6.3125]]]}, "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.75]], [[8.8125]], [[6.3125]]], "outputs": [[[3.015625]], [[1.0]]]}, "language_model.model.layers.19.feed_forward.router": {"inputs": [[[23.25]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op": {"inputs": [[[23.25]]], "outputs": [[[35.0]], [[36.25]], [[19.75]], [[10.5]], [[54.0]], [[55.25]], [[23.625]], [[17.25]], [[60.0]], [[10.0]], [[21.75]], [[22.875]], [[25.375]], [[23.375]], [[26.125]], [[37.75]], [[7.8125]], [[11.25]], [[47.5]], [[31.375]], [[26.5]], [[5.8125]], [[70.0]], [[24.875]], [[15.3125]], [[34.5]], [[31.875]], [[22.375]], [[63.0]], [[29.875]], [[37.75]], [[12.1875]], [[33.0]], [[43.0]], [[49.5]], [[30.0]], [[53.0]], [[21.75]], [[29.625]], [[33.75]], [[16.75]], [[48.5]], [[8.8125]], [[19.5]], [[60.0]], [[27.625]], [[14.9375]], [[12.9375]], [[12.0]], [[27.25]], [[46.25]], [[45.75]], [[32.5]], [[6.25]], [[41.0]], [[49.75]], [[30.625]], [[19.125]], [[25.625]], [[43.25]], [[21.875]], [[33.75]], [[42.75]], [[7.625]], [[18.5]], [[25.75]], [[43.25]], [[36.5]], [[16.875]], [[26.75]], [[9.5625]], [[45.5]], [[24.0]], [[58.75]], [[31.625]], [[29.0]], [[12.375]], [[33.75]], [[42.5]], [[53.0]], [[14.5625]], [[43.5]], [[27.25]], [[38.75]], [[77.0]], [[29.125]], [[33.0]], [[8.125]], [[5.65625]], [[30.375]], [[25.0]], [[33.5]], [[38.5]], [[28.625]], [[28.5]], [[22.25]], [[30.25]], [[27.75]], [[35.25]], [[56.5]], [[13.5]], [[79.0]], [[23.25]], [[20.375]], [[28.625]], [[45.0]], [[50.75]], [[28.25]], [[22.125]], [[47.75]], [[21.875]], [[16.625]], [[25.0]], [[50.25]], [[49.75]], [[8.1875]], [[10.0]], [[22.75]], [[33.25]], [[33.75]], [[52.75]], [[20.125]], [[9.125]], [[13.3125]], [[42.75]], [[10.25]], [[12.125]], [[45.25]], [[43.0]]]}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.7109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.466796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.6015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.25]]], "params": {"weight": [[0.32421875]]}}, "language_model.model.layers.19.feed_forward.shared_expert.down_proj": {"inputs": [[[21.625]]], "outputs": [[[11.0]], [[2.007134273635223e+38]]], "params": {"weight": [[0.72265625]]}}, "language_model.model.layers.20.self_attn.qkv_proj": {"inputs": [[[52.5]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.20.self_attn.o_proj": {"inputs": [[[4.78125]]], "outputs": [[[9.5]], [[18.375]]], "params": {"weight": [[0.8046875]]}}, "language_model.model.layers.20.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.86328125]], [[15.5]]]}, "language_model.model.layers.20.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.84375]]]}, "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.86328125]]]}, "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.15625]]]}, "language_model.model.layers.20.self_attn.attn.impl.k_cache": {"inputs": [[[15.5]]]}, "language_model.model.layers.20.self_attn.attn.impl.v_cache": {"inputs": [[[6.84375]]]}, "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.5625]], [[15.5]], [[6.84375]]], "outputs": [[[4.78125]], [[1.0]]]}, "language_model.model.layers.20.feed_forward.gate_up_proj": {"inputs": [[[41.5]]], "params": {"weight": [[0.365234375]]}}, "language_model.model.layers.20.feed_forward.down_proj": {"inputs": [[[15.5625]]], "outputs": [[[9.4375]], [[38.5]]], "params": {"weight": [[0.443359375]]}}, "language_model.model.layers.21.self_attn.qkv_proj": {"inputs": [[[56.25]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.21.self_attn.o_proj": {"inputs": [[[4.0]]], "outputs": [[[25.125]], [[37.75]]], "params": {"weight": [[0.431640625]]}}, "language_model.model.layers.21.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1171875]], [[13.625]]]}, "language_model.model.layers.21.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.625]]]}, "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1171875]]]}, "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.0]]]}, "language_model.model.layers.21.self_attn.attn.impl.k_cache": {"inputs": [[[13.625]]]}, "language_model.model.layers.21.self_attn.attn.impl.v_cache": {"inputs": [[[8.625]]]}, "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.3125]], [[13.625]], [[8.625]]], "outputs": [[[3.640625]], [[1.0]]]}, "language_model.model.layers.21.feed_forward.router": {"inputs": [[[24.5]]], "params": {"weight": [[0.45703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op": {"inputs": [[[24.5]]], "outputs": [[[6.625]], [[22.0]], [[48.0]], [[47.5]], [[29.0]], [[18.875]], [[77.0]], [[30.5]], [[21.75]], [[33.0]], [[39.5]], [[14.5]], [[10.375]], [[19.25]], [[52.5]], [[21.0]], [[51.25]], [[26.75]], [[26.125]], [[50.25]], [[40.25]], [[37.75]], [[29.75]], [[20.375]], [[20.5]], [[37.5]], [[7.71875]], [[9.125]], [[10.5625]], [[10.375]], [[18.75]], [[9.4375]], [[45.75]], [[35.0]], [[16.625]], [[21.875]], [[7.5625]], [[33.25]], [[24.375]], [[35.25]], [[25.0]], [[7.9375]], [[28.125]], [[15.4375]], [[6.21875]], [[46.25]], [[25.25]], [[10.875]], [[34.75]], [[33.75]], [[26.625]], [[44.0]], [[38.25]], [[31.5]], [[24.5]], [[8.3125]], [[33.25]], [[52.75]], [[15.8125]], [[53.25]], [[46.5]], [[38.75]], [[18.25]], [[27.0]], [[104.0]], [[20.375]], [[89.5]], [[12.8125]], [[31.875]], [[8.8125]], [[13.9375]], [[15.625]], [[22.0]], [[24.125]], [[28.5]], [[18.625]], [[9.5625]], [[40.5]], [[19.625]], [[25.25]], [[10.5]], [[34.5]], [[52.75]], [[33.25]], [[38.75]], [[40.5]], [[38.75]], [[7.875]], [[20.0]], [[7.0]], [[27.25]], [[28.625]], [[7.0625]], [[40.0]], [[18.625]], [[23.25]], [[26.125]], [[26.0]], [[25.125]], [[20.375]], [[29.125]], [[20.25]], [[70.5]], [[36.25]], [[55.0]], [[21.875]], [[43.5]], [[20.875]], [[35.0]], [[45.0]], [[36.75]], [[44.5]], [[52.5]], [[14.5625]], [[16.875]], [[30.375]], [[15.1875]], [[53.25]], [[18.25]], [[32.25]], [[30.875]], [[12.125]], [[18.125]], [[23.125]], [[23.5]], [[14.0]], [[75.0]], [[18.25]], [[58.5]]]}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.470703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.4296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[24.5]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.21.feed_forward.shared_expert.down_proj": {"inputs": [[[22.0]]], "outputs": [[[4.03125]], [[2.007134273635223e+38]]], "params": {"weight": [[0.47265625]]}}, "language_model.model.layers.22.self_attn.qkv_proj": {"inputs": [[[55.75]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.22.self_attn.o_proj": {"inputs": [[[4.3125]]], "outputs": [[[17.25]], [[23.25]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.22.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.09375]], [[15.5625]]]}, "language_model.model.layers.22.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.28125]]]}, "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.09375]]]}, "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.46875]]]}, "language_model.model.layers.22.self_attn.attn.impl.k_cache": {"inputs": [[[15.5625]]]}, "language_model.model.layers.22.self_attn.attn.impl.v_cache": {"inputs": [[[6.28125]]]}, "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.8125]], [[15.5625]], [[6.28125]]], "outputs": [[[4.3125]], [[1.0]]]}, "language_model.model.layers.22.feed_forward.gate_up_proj": {"inputs": [[[44.75]]], "params": {"weight": [[0.38671875]]}}, "language_model.model.layers.22.feed_forward.down_proj": {"inputs": [[[15.5625]]], "outputs": [[[12.0]], [[80.5]]], "params": {"weight": [[0.60546875]]}}, "language_model.model.layers.23.self_attn.qkv_proj": {"inputs": [[[67.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.23.self_attn.o_proj": {"inputs": [[[7.46875]]], "outputs": [[[50.5]], [[61.5]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.23.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.98828125]], [[15.6875]]]}, "language_model.model.layers.23.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.375]]]}, "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.98828125]]]}, "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.53125]]]}, "language_model.model.layers.23.self_attn.attn.impl.k_cache": {"inputs": [[[15.6875]]]}, "language_model.model.layers.23.self_attn.attn.impl.v_cache": {"inputs": [[[11.375]]]}, "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.5]], [[15.6875]], [[11.375]]], "outputs": [[[7.46875]], [[1.0]]]}, "language_model.model.layers.23.feed_forward.router": {"inputs": [[[23.625]]], "params": {"weight": [[0.474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op": {"inputs": [[[23.625]]], "outputs": [[[59.25]], [[87.0]], [[7.5625]], [[39.0]], [[47.0]], [[18.625]], [[40.5]], [[27.625]], [[41.75]], [[97.0]], [[40.75]], [[18.25]], [[68.5]], [[49.25]], [[16.625]], [[32.5]], [[47.25]], [[98.5]], [[16.75]], [[29.5]], [[26.5]], [[26.625]], [[40.75]], [[12.375]], [[13.6875]], [[43.0]], [[17.0]], [[30.875]], [[94.0]], [[12.75]], [[16.125]], [[26.375]], [[30.625]], [[13.8125]], [[17.625]], [[14.125]], [[28.625]], [[28.375]], [[17.5]], [[14.4375]], [[48.0]], [[56.0]], [[8.875]], [[45.0]], [[9.1875]], [[34.0]], [[9.25]], [[71.0]], [[73.0]], [[8.375]], [[87.0]], [[17.25]], [[50.5]], [[36.75]], [[23.0]], [[27.375]], [[31.75]], [[100.0]], [[10.5]], [[61.5]], [[24.375]], [[45.0]], [[46.75]], [[9.3125]], [[6.90625]], [[11.5625]], [[13.625]], [[34.75]], [[40.25]], [[43.0]], [[62.25]], [[9.3125]], [[25.75]], [[60.0]], [[57.75]], [[15.5625]], [[46.25]], [[27.125]], [[45.75]], [[24.25]], [[25.625]], [[23.75]], [[19.5]], [[49.5]], [[23.875]], [[9.75]], [[45.5]], [[14.6875]], [[10.625]], [[33.0]], [[18.0]], [[30.0]], [[55.5]], [[45.0]], [[10.4375]], [[16.875]], [[36.5]], [[25.75]], [[34.5]], [[18.25]], [[30.0]], [[11.0625]], [[20.625]], [[9.25]], [[15.9375]], [[70.0]], [[50.0]], [[16.5]], [[50.25]], [[8.75]], [[75.0]], [[14.25]], [[103.5]], [[10.4375]], [[82.0]], [[32.0]], [[101.5]], [[22.125]], [[54.0]], [[14.8125]], [[14.75]], [[13.75]], [[44.75]], [[54.0]], [[55.5]], [[49.5]], [[43.75]], [[11.4375]], [[12.25]]]}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.3984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.59765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.53125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.625]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.23.feed_forward.shared_expert.down_proj": {"inputs": [[[30.25]]], "outputs": [[[5.8125]], [[2.007134273635223e+38]]], "params": {"weight": [[0.5]]}}, "language_model.model.layers.24.self_attn.qkv_proj": {"inputs": [[[60.5]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.24.self_attn.o_proj": {"inputs": [[[6.09375]]], "outputs": [[[36.0]], [[42.5]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.24.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.82421875]], [[14.6875]]]}, "language_model.model.layers.24.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.0]]]}, "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.82421875]]]}, "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.1875]]]}, "language_model.model.layers.24.self_attn.attn.impl.k_cache": {"inputs": [[[14.6875]]]}, "language_model.model.layers.24.self_attn.attn.impl.v_cache": {"inputs": [[[12.0]]]}, "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.375]], [[14.6875]], [[12.0]]], "outputs": [[[6.09375]], [[1.0]]]}, "language_model.model.layers.24.feed_forward.gate_up_proj": {"inputs": [[[40.5]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.24.feed_forward.down_proj": {"inputs": [[[21.75]]], "outputs": [[[13.3125]], [[62.5]]], "params": {"weight": [[0.59375]]}}, "language_model.model.layers.25.self_attn.qkv_proj": {"inputs": [[[49.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.25.self_attn.o_proj": {"inputs": [[[5.625]]], "outputs": [[[47.75]], [[57.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.25.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.03125]], [[24.125]]]}, "language_model.model.layers.25.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.53125]]]}, "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.03125]]]}, "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.46875]]]}, "language_model.model.layers.25.self_attn.attn.impl.k_cache": {"inputs": [[[24.125]]]}, "language_model.model.layers.25.self_attn.attn.impl.v_cache": {"inputs": [[[6.53125]]]}, "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.4375]], [[24.125]], [[6.53125]]], "outputs": [[[5.625]], [[1.0]]]}, "language_model.model.layers.25.feed_forward.router": {"inputs": [[[30.375]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op": {"inputs": [[[30.375]]], "outputs": [[[7.5625]], [[63.25]], [[30.875]], [[20.75]], [[9.375]], [[32.25]], [[42.5]], [[22.75]], [[68.0]], [[10.5625]], [[12.125]], [[8.5625]], [[13.9375]], [[32.5]], [[50.5]], [[43.75]], [[24.75]], [[18.125]], [[21.25]], [[40.75]], [[33.75]], [[35.0]], [[41.0]], [[42.75]], [[29.5]], [[6.90625]], [[14.875]], [[48.5]], [[18.875]], [[13.875]], [[62.0]], [[21.0]], [[9.5]], [[9.6875]], [[103.5]], [[27.0]], [[11.9375]], [[17.25]], [[41.75]], [[17.25]], [[50.75]], [[24.0]], [[47.75]], [[36.25]], [[42.25]], [[16.125]], [[62.75]], [[92.0]], [[10.1875]], [[19.5]], [[118.0]], [[36.75]], [[14.3125]], [[35.75]], [[14.875]], [[23.625]], [[55.75]], [[31.0]], [[15.0625]], [[7.3125]], [[33.75]], [[26.875]], [[41.0]], [[32.0]], [[17.125]], [[97.0]], [[10.375]], [[42.25]], [[23.5]], [[26.75]], [[14.5]], [[13.625]], [[28.25]], [[27.25]], [[36.75]], [[86.0]], [[59.5]], [[13.8125]], [[12.5625]], [[29.25]], [[34.5]], [[16.5]], [[29.625]], [[29.625]], [[34.25]], [[8.8125]], [[17.125]], [[41.0]], [[53.25]], [[21.125]], [[42.5]], [[11.25]], [[26.875]], [[30.625]], [[22.875]], [[9.875]], [[39.25]], [[34.25]], [[52.0]], [[9.625]], [[36.0]], [[15.3125]], [[22.375]], [[24.0]], [[15.6875]], [[18.25]], [[20.5]], [[36.5]], [[5.0]], [[44.75]], [[30.625]], [[48.5]], [[27.0]], [[49.75]], [[37.75]], [[27.75]], [[44.5]], [[67.0]], [[13.75]], [[78.5]], [[46.0]], [[29.0]], [[56.5]], [[29.25]], [[12.375]], [[46.75]], [[6.53125]], [[17.75]], [[9.0]]]}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.357421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[30.375]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.25.feed_forward.shared_expert.down_proj": {"inputs": [[[38.25]]], "outputs": [[[13.75]], [[2.007134273635223e+38]]], "params": {"weight": [[0.51171875]]}}, "language_model.model.layers.26.self_attn.qkv_proj": {"inputs": [[[63.25]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.26.self_attn.o_proj": {"inputs": [[[5.65625]]], "outputs": [[[29.0]], [[53.0]]], "params": {"weight": [[0.765625]]}}, "language_model.model.layers.26.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1953125]], [[15.75]]]}, "language_model.model.layers.26.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.125]]]}, "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1953125]]]}, "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.875]]]}, "language_model.model.layers.26.self_attn.attn.impl.k_cache": {"inputs": [[[15.75]]]}, "language_model.model.layers.26.self_attn.attn.impl.v_cache": {"inputs": [[[8.125]]]}, "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.6875]], [[15.75]], [[8.125]]], "outputs": [[[5.65625]], [[1.0]]]}, "language_model.model.layers.26.feed_forward.gate_up_proj": {"inputs": [[[45.5]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.26.feed_forward.down_proj": {"inputs": [[[24.5]]], "outputs": [[[15.25]], [[103.0]]], "params": {"weight": [[0.78515625]]}}, "language_model.model.layers.27.self_attn.qkv_proj": {"inputs": [[[72.5]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.27.self_attn.o_proj": {"inputs": [[[3.40625]]], "outputs": [[[60.0]], [[66.0]]], "params": {"weight": [[0.73046875]]}}, "language_model.model.layers.27.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.6640625]], [[10.375]]]}, "language_model.model.layers.27.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.90625]]]}, "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.6640625]]]}, "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.375]]]}, "language_model.model.layers.27.self_attn.attn.impl.k_cache": {"inputs": [[[10.375]]]}, "language_model.model.layers.27.self_attn.attn.impl.v_cache": {"inputs": [[[4.90625]]]}, "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[7.96875]], [[10.375]], [[4.90625]]], "outputs": [[[3.40625]], [[1.0]]]}, "language_model.model.layers.27.feed_forward.router": {"inputs": [[[30.875]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op": {"inputs": [[[30.875]]], "outputs": [[[7.25]], [[22.75]], [[45.0]], [[40.75]], [[12.1875]], [[11.875]], [[66.0]], [[46.25]], [[23.125]], [[38.5]], [[28.5]], [[22.0]], [[40.75]], [[54.0]], [[12.5625]], [[64.5]], [[29.25]], [[17.0]], [[98.0]], [[27.25]], [[22.75]], [[21.375]], [[13.3125]], [[22.25]], [[19.875]], [[24.75]], [[9.0625]], [[45.75]], [[19.125]], [[17.0]], [[52.25]], [[81.5]], [[13.1875]], [[59.5]], [[16.375]], [[82.0]], [[14.0625]], [[24.625]], [[65.0]], [[35.5]], [[15.25]], [[34.75]], [[43.0]], [[28.0]], [[29.375]], [[10.625]], [[34.0]], [[11.8125]], [[29.375]], [[36.75]], [[16.875]], [[37.0]], [[11.6875]], [[48.5]], [[40.5]], [[28.75]], [[43.25]], [[40.5]], [[27.0]], [[60.25]], [[10.0625]], [[33.25]], [[49.25]], [[44.0]], [[24.0]], [[48.25]], [[120.5]], [[11.1875]], [[14.8125]], [[19.125]], [[22.0]], [[21.5]], [[43.5]], [[99.0]], [[32.25]], [[27.375]], [[84.5]], [[38.0]], [[31.75]], [[35.25]], [[42.25]], [[41.5]], [[28.0]], [[23.625]], [[33.5]], [[53.25]], [[25.75]], [[15.3125]], [[29.25]], [[12.3125]], [[22.125]], [[12.4375]], [[60.0]], [[15.5]], [[34.0]], [[29.875]], [[65.5]], [[22.75]], [[48.5]], [[15.625]], [[22.875]], [[41.5]], [[35.5]], [[12.9375]], [[20.625]], [[12.4375]], [[18.375]], [[13.9375]], [[39.5]], [[13.9375]], [[52.5]], [[19.125]], [[55.0]], [[16.125]], [[15.8125]], [[23.375]], [[18.875]], [[27.0]], [[34.5]], [[28.5]], [[62.25]], [[11.0]], [[61.5]], [[21.75]], [[103.0]], [[13.625]], [[32.5]], [[18.875]], [[33.75]]]}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[30.875]]], "params": {"weight": [[0.357421875]]}}, "language_model.model.layers.27.feed_forward.shared_expert.down_proj": {"inputs": [[[32.75]]], "outputs": [[[8.75]], [[2.007134273635223e+38]]], "params": {"weight": [[0.5859375]]}}, "language_model.model.layers.28.self_attn.qkv_proj": {"inputs": [[[77.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.28.self_attn.o_proj": {"inputs": [[[9.5625]]], "outputs": [[[72.0]], [[74.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.28.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.328125]], [[17.875]]]}, "language_model.model.layers.28.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.5625]]]}, "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.328125]]]}, "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.0]]]}, "language_model.model.layers.28.self_attn.attn.impl.k_cache": {"inputs": [[[17.875]]]}, "language_model.model.layers.28.self_attn.attn.impl.v_cache": {"inputs": [[[11.5625]]]}, "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.0]], [[17.875]], [[11.5625]]], "outputs": [[[9.5625]], [[1.0]]]}, "language_model.model.layers.28.feed_forward.gate_up_proj": {"inputs": [[[40.25]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.28.feed_forward.down_proj": {"inputs": [[[119.5]]], "outputs": [[[95.0]], [[103.5]]], "params": {"weight": [[0.78125]]}}, "language_model.model.layers.29.self_attn.qkv_proj": {"inputs": [[[66.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.29.self_attn.o_proj": {"inputs": [[[10.3125]]], "outputs": [[[70.5]], [[76.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.29.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.015625]], [[21.375]]]}, "language_model.model.layers.29.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.3125]]]}, "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.015625]]]}, "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.3125]]]}, "language_model.model.layers.29.self_attn.attn.impl.k_cache": {"inputs": [[[21.375]]]}, "language_model.model.layers.29.self_attn.attn.impl.v_cache": {"inputs": [[[11.3125]]]}, "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.75]], [[21.375]], [[11.3125]]], "outputs": [[[10.3125]], [[1.0]]]}, "language_model.model.layers.29.feed_forward.router": {"inputs": [[[31.75]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op": {"inputs": [[[31.75]]], "outputs": [[[6.90625]], [[9.0]], [[43.5]], [[43.75]], [[35.75]], [[20.125]], [[47.0]], [[9.25]], [[68.0]], [[27.125]], [[25.875]], [[13.3125]], [[15.25]], [[11.375]], [[42.5]], [[24.625]], [[8.4375]], [[60.0]], [[31.5]], [[30.75]], [[12.375]], [[32.75]], [[17.375]], [[31.375]], [[17.75]], [[43.25]], [[27.5]], [[31.5]], [[30.0]], [[15.125]], [[46.75]], [[37.25]], [[18.375]], [[51.0]], [[54.75]], [[17.0]], [[43.5]], [[32.25]], [[27.0]], [[107.5]], [[53.0]], [[16.375]], [[23.625]], [[13.9375]], [[73.0]], [[32.25]], [[26.375]], [[22.875]], [[24.75]], [[31.375]], [[13.625]], [[22.875]], [[24.5]], [[55.0]], [[63.25]], [[21.375]], [[39.25]], [[45.75]], [[59.75]], [[26.75]], [[20.0]], [[37.5]], [[11.4375]], [[21.875]], [[17.875]], [[40.25]], [[71.0]], [[47.75]], [[33.25]], [[42.25]], [[16.0]], [[19.875]], [[43.25]], [[26.5]], [[33.0]], [[58.75]], [[63.25]], [[20.625]], [[35.75]], [[35.25]], [[19.0]], [[34.0]], [[20.5]], [[13.5625]], [[81.5]], [[15.0625]], [[34.5]], [[34.5]], [[10.0]], [[48.0]], [[17.625]], [[35.0]], [[32.25]], [[17.625]], [[64.5]], [[6.40625]], [[31.875]], [[15.5]], [[15.625]], [[43.75]], [[13.5625]], [[39.25]], [[24.625]], [[19.375]], [[46.0]], [[8.5625]], [[80.5]], [[68.5]], [[17.75]], [[19.0]], [[43.75]], [[27.125]], [[45.5]], [[53.5]], [[12.8125]], [[37.0]], [[36.75]], [[31.125]], [[37.25]], [[87.5]], [[18.5]], [[41.0]], [[22.25]], [[51.75]], [[93.5]], [[15.8125]], [[30.0]], [[105.0]], [[25.0]]]}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.6171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[31.75]]], "params": {"weight": [[0.44921875]]}}, "language_model.model.layers.29.feed_forward.shared_expert.down_proj": {"inputs": [[[37.75]]], "outputs": [[[12.8125]], [[2.007134273635223e+38]]], "params": {"weight": [[0.55859375]]}}, "language_model.model.layers.30.self_attn.qkv_proj": {"inputs": [[[71.5]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.30.self_attn.o_proj": {"inputs": [[[5.625]]], "outputs": [[[52.25]], [[57.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.30.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0]], [[16.75]]]}, "language_model.model.layers.30.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.25]]]}, "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0]]]}, "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.8125]]]}, "language_model.model.layers.30.self_attn.attn.impl.k_cache": {"inputs": [[[16.75]]]}, "language_model.model.layers.30.self_attn.attn.impl.v_cache": {"inputs": [[[7.25]]]}, "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.125]], [[16.75]], [[7.25]]], "outputs": [[[5.625]], [[1.0]]]}, "language_model.model.layers.30.feed_forward.gate_up_proj": {"inputs": [[[32.0]]], "params": {"weight": [[0.515625]]}}, "language_model.model.layers.30.feed_forward.down_proj": {"inputs": [[[33.0]]], "outputs": [[[18.625]], [[139.0]]], "params": {"weight": [[0.5234375]]}}, "language_model.model.layers.31.self_attn.qkv_proj": {"inputs": [[[71.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.31.self_attn.o_proj": {"inputs": [[[5.96875]]], "outputs": [[[66.5]], [[88.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.31.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.9609375]], [[17.875]]]}, "language_model.model.layers.31.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.0]]]}, "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.9609375]]]}, "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.03125]]]}, "language_model.model.layers.31.self_attn.attn.impl.k_cache": {"inputs": [[[17.875]]]}, "language_model.model.layers.31.self_attn.attn.impl.v_cache": {"inputs": [[[12.0]]]}, "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.875]], [[17.875]], [[12.0]]], "outputs": [[[5.96875]], [[1.0]]]}, "language_model.model.layers.31.feed_forward.router": {"inputs": [[[33.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op": {"inputs": [[[33.0]]], "outputs": [[[8.0625]], [[11.625]], [[90.0]], [[22.875]], [[36.0]], [[44.25]], [[44.75]], [[83.0]], [[18.0]], [[33.75]], [[85.0]], [[49.0]], [[13.5]], [[18.625]], [[13.875]], [[39.0]], [[22.75]], [[26.875]], [[72.5]], [[107.5]], [[33.25]], [[9.8125]], [[18.75]], [[38.0]], [[79.0]], [[53.5]], [[38.75]], [[56.5]], [[35.25]], [[23.25]], [[22.875]], [[20.5]], [[24.125]], [[28.25]], [[42.75]], [[15.625]], [[18.625]], [[27.75]], [[30.625]], [[49.0]], [[59.25]], [[65.0]], [[60.0]], [[14.25]], [[59.0]], [[49.75]], [[60.25]], [[26.125]], [[59.0]], [[35.5]], [[62.0]], [[46.0]], [[37.0]], [[63.5]], [[37.5]], [[41.75]], [[31.625]], [[35.0]], [[51.25]], [[27.375]], [[17.75]], [[23.0]], [[46.0]], [[27.25]], [[48.0]], [[82.0]], [[35.0]], [[35.0]], [[58.25]], [[31.875]], [[26.5]], [[68.5]], [[61.25]], [[24.375]], [[66.5]], [[13.5]], [[11.9375]], [[49.5]], [[52.75]], [[25.125]], [[42.0]], [[25.625]], [[82.0]], [[35.5]], [[63.0]], [[13.375]], [[54.25]], [[63.5]], [[17.625]], [[33.75]], [[69.5]], [[53.75]], [[34.0]], [[51.75]], [[11.625]], [[48.25]], [[72.5]], [[40.75]], [[15.875]], [[23.25]], [[32.25]], [[16.125]], [[10.1875]], [[46.5]], [[37.5]], [[35.0]], [[13.75]], [[41.0]], [[12.9375]], [[19.875]], [[67.0]], [[44.75]], [[122.0]], [[25.875]], [[32.5]], [[39.0]], [[50.25]], [[66.5]], [[22.625]], [[19.625]], [[78.5]], [[31.75]], [[21.625]], [[14.375]], [[13.0]], [[27.0]], [[16.25]], [[30.375]], [[27.125]]]}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[33.0]]], "params": {"weight": [[0.53515625]]}}, "language_model.model.layers.31.feed_forward.shared_expert.down_proj": {"inputs": [[[65.5]]], "outputs": [[[31.0]], [[2.007134273635223e+38]]], "params": {"weight": [[0.90625]]}}, "language_model.model.layers.32.self_attn.qkv_proj": {"inputs": [[[53.25]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.32.self_attn.o_proj": {"inputs": [[[6.0625]]], "outputs": [[[96.0]], [[105.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.32.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.28125]], [[19.125]]]}, "language_model.model.layers.32.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.1875]]]}, "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.28125]]]}, "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.40625]]]}, "language_model.model.layers.32.self_attn.attn.impl.k_cache": {"inputs": [[[19.125]]]}, "language_model.model.layers.32.self_attn.attn.impl.v_cache": {"inputs": [[[7.1875]]]}, "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[17.125]], [[19.125]], [[7.1875]]], "outputs": [[[6.0625]], [[1.0]]]}, "language_model.model.layers.32.feed_forward.gate_up_proj": {"inputs": [[[32.75]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.32.feed_forward.down_proj": {"inputs": [[[45.5]]], "outputs": [[[21.125]], [[79.5]]], "params": {"weight": [[0.83203125]]}}, "language_model.model.layers.33.self_attn.qkv_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.33.self_attn.o_proj": {"inputs": [[[6.8125]]], "outputs": [[[97.0]], [[118.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.33.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.265625]], [[19.0]]]}, "language_model.model.layers.33.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.0]]]}, "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.265625]]]}, "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.8125]]]}, "language_model.model.layers.33.self_attn.attn.impl.k_cache": {"inputs": [[[19.0]]]}, "language_model.model.layers.33.self_attn.attn.impl.v_cache": {"inputs": [[[8.0]]]}, "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.0]], [[19.0]], [[8.0]]], "outputs": [[[6.0625]], [[1.0]]]}, "language_model.model.layers.33.feed_forward.router": {"inputs": [[[64.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op": {"inputs": [[[64.0]]], "outputs": [[[8.875]], [[57.25]], [[57.0]], [[61.25]], [[48.25]], [[33.5]], [[65.5]], [[17.25]], [[42.25]], [[38.0]], [[72.5]], [[19.875]], [[63.25]], [[74.0]], [[19.25]], [[78.0]], [[34.0]], [[28.25]], [[25.0]], [[18.125]], [[23.75]], [[22.125]], [[17.375]], [[51.0]], [[37.0]], [[26.0]], [[30.875]], [[105.0]], [[35.25]], [[139.0]], [[58.75]], [[27.0]], [[58.75]], [[18.25]], [[87.5]], [[31.5]], [[24.375]], [[47.0]], [[16.375]], [[64.5]], [[31.125]], [[18.875]], [[38.75]], [[113.0]], [[85.5]], [[40.25]], [[20.0]], [[75.0]], [[16.875]], [[58.25]], [[63.5]], [[41.5]], [[10.75]], [[31.25]], [[19.75]], [[127.5]], [[89.0]], [[77.5]], [[19.625]], [[42.5]], [[92.5]], [[36.25]], [[73.0]], [[48.75]], [[108.0]], [[22.5]], [[76.0]], [[27.75]], [[17.0]], [[55.25]], [[32.75]], [[66.0]], [[47.0]], [[73.5]], [[102.0]], [[19.875]], [[55.25]], [[43.25]], [[18.875]], [[13.875]], [[34.0]], [[18.375]], [[11.75]], [[39.0]], [[37.75]], [[15.5]], [[89.0]], [[58.25]], [[39.75]], [[65.5]], [[46.25]], [[16.5]], [[22.125]], [[21.5]], [[35.5]], [[46.5]], [[77.0]], [[21.375]], [[62.75]], [[40.75]], [[56.0]], [[80.5]], [[80.0]], [[39.0]], [[36.75]], [[72.0]], [[19.5]], [[73.0]], [[29.25]], [[19.5]], [[11.8125]], [[34.5]], [[117.5]], [[25.25]], [[42.25]], [[79.5]], [[31.75]], [[16.75]], [[38.0]], [[23.25]], [[92.5]], [[42.5]], [[31.0]], [[71.5]], [[9.9375]], [[44.0]], [[23.375]], [[16.5]], [[30.625]]]}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.357421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.384765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[64.0]]], "params": {"weight": [[0.443359375]]}}, "language_model.model.layers.33.feed_forward.shared_expert.down_proj": {"inputs": [[[118.0]]], "outputs": [[[40.75]], [[2.007134273635223e+38]]], "params": {"weight": [[0.8671875]]}}, "language_model.model.layers.34.self_attn.qkv_proj": {"inputs": [[[54.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.34.self_attn.o_proj": {"inputs": [[[8.0625]]], "outputs": [[[46.75]], [[80.5]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.34.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.140625]], [[17.25]]]}, "language_model.model.layers.34.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.125]]]}, "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.140625]]]}, "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.5625]]]}, "language_model.model.layers.34.self_attn.attn.impl.k_cache": {"inputs": [[[17.25]]]}, "language_model.model.layers.34.self_attn.attn.impl.v_cache": {"inputs": [[[9.125]]]}, "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.4375]], [[17.25]], [[9.125]]], "outputs": [[[8.0625]], [[1.0]]]}, "language_model.model.layers.34.feed_forward.gate_up_proj": {"inputs": [[[31.125]]], "params": {"weight": [[0.41015625]]}}, "language_model.model.layers.34.feed_forward.down_proj": {"inputs": [[[60.0]]], "outputs": [[[30.375]], [[128.0]]], "params": {"weight": [[0.470703125]]}}, "language_model.model.layers.35.self_attn.qkv_proj": {"inputs": [[[52.75]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.self_attn.o_proj": {"inputs": [[[7.59375]]], "outputs": [[[87.5]], [[93.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.35.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.78515625]], [[12.6875]]]}, "language_model.model.layers.35.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.0625]]]}, "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.78515625]]]}, "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.65625]]]}, "language_model.model.layers.35.self_attn.attn.impl.k_cache": {"inputs": [[[12.6875]]]}, "language_model.model.layers.35.self_attn.attn.impl.v_cache": {"inputs": [[[9.8125]]]}, "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.625]], [[12.6875]], [[9.8125]]], "outputs": [[[7.59375]], [[1.0]]]}, "language_model.model.layers.35.feed_forward.router": {"inputs": [[[64.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op": {"inputs": [[[64.0]]], "outputs": [[[13.125]], [[130.0]], [[25.25]], [[12.3125]], [[42.5]], [[130.0]], [[24.75]], [[57.75]], [[103.5]], [[51.0]], [[38.25]], [[66.0]], [[24.75]], [[84.5]], [[44.5]], [[74.5]], [[57.0]], [[73.5]], [[46.75]], [[75.5]], [[47.25]], [[52.5]], [[37.25]], [[28.875]], [[48.75]], [[59.75]], [[57.5]], [[75.5]], [[82.5]], [[35.5]], [[52.75]], [[87.0]], [[58.75]], [[52.5]], [[31.125]], [[47.5]], [[93.0]], [[48.0]], [[38.0]], [[39.0]], [[71.5]], [[40.5]], [[58.0]], [[23.5]], [[126.0]], [[56.25]], [[24.0]], [[23.625]], [[30.875]], [[152.0]], [[83.0]], [[23.5]], [[40.75]], [[51.5]], [[27.0]], [[30.0]], [[31.0]], [[104.5]], [[30.625]], [[49.25]], [[50.0]], [[66.0]], [[25.0]], [[59.5]], [[43.25]], [[65.0]], [[18.0]], [[132.0]], [[30.375]], [[15.3125]], [[105.0]], [[31.75]], [[117.5]], [[185.0]], [[77.5]], [[43.5]], [[63.5]], [[41.75]], [[66.5]], [[70.5]], [[87.5]], [[62.75]], [[72.0]], [[17.125]], [[38.75]], [[65.0]], [[155.0]], [[47.5]], [[28.375]], [[18.5]], [[27.125]], [[125.0]], [[56.0]], [[63.75]], [[102.5]], [[69.5]], [[60.75]], [[54.25]], [[43.0]], [[132.0]], [[30.0]], [[119.5]], [[21.75]], [[42.75]], [[29.625]], [[29.5]], [[52.25]], [[35.5]], [[40.75]], [[108.0]], [[53.0]], [[55.75]], [[36.0]], [[47.25]], [[25.75]], [[28.5]], [[85.5]], [[47.75]], [[19.5]], [[35.75]], [[57.25]], [[39.25]], [[21.125]], [[26.75]], [[89.0]], [[75.0]], [[44.5]], [[26.875]], [[40.0]]]}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.419921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.490234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.408203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[64.0]]], "params": {"weight": [[0.416015625]]}}, "language_model.model.layers.35.feed_forward.shared_expert.down_proj": {"inputs": [[[80.0]]], "outputs": [[[9.6875]], [[2.007134273635223e+38]]], "params": {"weight": [[0.455078125]]}}, "language_model.model.layers.36.self_attn.qkv_proj": {"inputs": [[[56.75]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.36.self_attn.o_proj": {"inputs": [[[6.6875]]], "outputs": [[[27.75]], [[43.0]]], "params": {"weight": [[0.53125]]}}, "language_model.model.layers.36.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.125]], [[13.8125]]]}, "language_model.model.layers.36.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.25]]]}, "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.125]]]}, "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.0]]]}, "language_model.model.layers.36.self_attn.attn.impl.k_cache": {"inputs": [[[14.3125]]]}, "language_model.model.layers.36.self_attn.attn.impl.v_cache": {"inputs": [[[8.25]]]}, "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[16.125]], [[14.3125]], [[8.0625]]], "outputs": [[[6.6875]], [[1.0]]]}, "language_model.model.layers.36.feed_forward.gate_up_proj": {"inputs": [[[29.625]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.36.feed_forward.down_proj": {"inputs": [[[71.0]]], "outputs": [[[38.25]], [[76.5]]], "params": {"weight": [[0.7578125]]}}, "language_model.model.layers.37.self_attn.qkv_proj": {"inputs": [[[50.25]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.37.self_attn.o_proj": {"inputs": [[[10.875]]], "outputs": [[[102.0]], [[105.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.37.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.8359375]], [[12.875]]]}, "language_model.model.layers.37.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.5625]]]}, "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.8359375]]]}, "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[9.75]]]}, "language_model.model.layers.37.self_attn.attn.impl.k_cache": {"inputs": [[[12.875]]]}, "language_model.model.layers.37.self_attn.attn.impl.v_cache": {"inputs": [[[12.5625]]]}, "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[23.5]], [[12.875]], [[12.5625]]], "outputs": [[[10.875]], [[1.0]]]}, "language_model.model.layers.37.feed_forward.router": {"inputs": [[[70.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op": {"inputs": [[[70.0]]], "outputs": [[[13.5625]], [[97.0]], [[110.5]], [[29.75]], [[101.5]], [[53.0]], [[33.0]], [[56.75]], [[90.0]], [[26.875]], [[109.5]], [[23.875]], [[22.75]], [[33.25]], [[49.25]], [[57.25]], [[40.25]], [[42.5]], [[43.0]], [[52.0]], [[62.5]], [[47.25]], [[66.0]], [[71.0]], [[31.75]], [[70.0]], [[72.5]], [[39.5]], [[81.5]], [[138.0]], [[60.75]], [[21.25]], [[96.0]], [[63.25]], [[22.125]], [[76.5]], [[59.5]], [[96.0]], [[144.0]], [[88.0]], [[65.0]], [[65.5]], [[26.625]], [[36.0]], [[64.0]], [[54.5]], [[106.0]], [[110.0]], [[29.75]], [[53.75]], [[22.25]], [[27.5]], [[39.75]], [[84.5]], [[60.0]], [[44.5]], [[107.0]], [[201.0]], [[47.0]], [[81.0]], [[99.5]], [[87.5]], [[29.5]], [[27.25]], [[35.0]], [[54.5]], [[30.0]], [[32.25]], [[96.5]], [[98.0]], [[31.125]], [[52.25]], [[66.0]], [[53.75]], [[16.125]], [[33.75]], [[70.5]], [[124.0]], [[22.875]], [[61.25]], [[114.0]], [[63.0]], [[31.125]], [[117.0]], [[27.5]], [[55.5]], [[36.25]], [[94.0]], [[57.25]], [[104.5]], [[100.5]], [[34.5]], [[74.5]], [[30.875]], [[38.0]], [[27.5]], [[62.75]], [[19.0]], [[29.75]], [[73.0]], [[70.5]], [[148.0]], [[63.0]], [[58.75]], [[56.5]], [[85.0]], [[88.0]], [[88.0]], [[22.625]], [[52.25]], [[46.5]], [[45.0]], [[62.0]], [[48.5]], [[143.0]], [[73.0]], [[74.0]], [[57.25]], [[122.5]], [[46.25]], [[104.5]], [[22.5]], [[91.5]], [[63.0]], [[64.5]], [[77.5]], [[31.25]], [[85.5]], [[107.5]]]}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[70.0]]], "params": {"weight": [[0.341796875]]}}, "language_model.model.layers.37.feed_forward.shared_expert.down_proj": {"inputs": [[[96.5]]], "outputs": [[[13.5]], [[2.007134273635223e+38]]], "params": {"weight": [[0.58203125]]}}, "language_model.model.layers.38.self_attn.qkv_proj": {"inputs": [[[47.25]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.38.self_attn.o_proj": {"inputs": [[[6.71875]]], "outputs": [[[26.75]], [[56.0]]], "params": {"weight": [[0.515625]]}}, "language_model.model.layers.38.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.609375]], [[18.0]]]}, "language_model.model.layers.38.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.875]]]}, "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.609375]]]}, "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.1875]]]}, "language_model.model.layers.38.self_attn.attn.impl.k_cache": {"inputs": [[[18.0]]]}, "language_model.model.layers.38.self_attn.attn.impl.v_cache": {"inputs": [[[8.875]]]}, "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[22.125]], [[18.0]], [[8.875]]], "outputs": [[[6.71875]], [[1.0]]]}, "language_model.model.layers.38.feed_forward.gate_up_proj": {"inputs": [[[31.125]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.38.feed_forward.down_proj": {"inputs": [[[90.5]]], "outputs": [[[24.75]], [[193.0]]], "params": {"weight": [[0.6015625]]}}, "language_model.model.layers.39.self_attn.qkv_proj": {"inputs": [[[49.75]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.39.self_attn.o_proj": {"inputs": [[[6.21875]]], "outputs": [[[129.0]], [[134.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.39.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.984375]], [[12.625]]]}, "language_model.model.layers.39.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.375]]]}, "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.984375]]]}, "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.15625]]]}, "language_model.model.layers.39.self_attn.attn.impl.k_cache": {"inputs": [[[12.625]]]}, "language_model.model.layers.39.self_attn.attn.impl.v_cache": {"inputs": [[[7.375]]]}, "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.25]], [[12.625]], [[7.375]]], "outputs": [[[6.21875]], [[1.0]]]}, "language_model.model.layers.39.feed_forward.router": {"inputs": [[[61.5]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op": {"inputs": [[[61.5]]], "outputs": [[[11.5625]], [[47.25]], [[60.5]], [[86.5]], [[78.5]], [[66.0]], [[35.5]], [[40.0]], [[80.0]], [[39.75]], [[50.0]], [[85.0]], [[62.0]], [[132.0]], [[106.0]], [[54.0]], [[53.5]], [[79.0]], [[142.0]], [[47.25]], [[54.75]], [[83.5]], [[118.5]], [[36.25]], [[52.75]], [[70.5]], [[128.0]], [[102.0]], [[128.0]], [[64.5]], [[44.25]], [[62.25]], [[83.0]], [[27.375]], [[30.75]], [[88.5]], [[56.75]], [[173.0]], [[102.5]], [[98.0]], [[59.0]], [[73.5]], [[53.75]], [[81.0]], [[36.0]], [[72.0]], [[33.75]], [[47.5]], [[53.25]], [[130.0]], [[59.5]], [[91.0]], [[72.5]], [[57.5]], [[52.25]], [[91.0]], [[104.0]], [[40.25]], [[69.0]], [[42.0]], [[38.75]], [[118.5]], [[102.5]], [[119.5]], [[27.875]], [[29.5]], [[68.5]], [[39.0]], [[130.0]], [[105.5]], [[112.5]], [[90.0]], [[40.5]], [[129.0]], [[144.0]], [[65.5]], [[64.0]], [[52.0]], [[105.0]], [[151.0]], [[35.25]], [[45.25]], [[117.5]], [[56.75]], [[46.5]], [[90.5]], [[103.0]], [[176.0]], [[56.0]], [[77.5]], [[70.5]], [[68.5]], [[121.0]], [[27.625]], [[41.75]], [[56.75]], [[43.0]], [[69.5]], [[97.0]], [[61.0]], [[154.0]], [[82.0]], [[66.5]], [[99.0]], [[108.0]], [[75.5]], [[127.0]], [[110.5]], [[137.0]], [[78.5]], [[55.5]], [[59.5]], [[70.5]], [[38.5]], [[85.5]], [[95.5]], [[95.0]], [[43.75]], [[34.5]], [[76.0]], [[92.0]], [[40.75]], [[75.0]], [[123.5]], [[47.5]], [[87.0]], [[99.0]], [[79.5]], [[93.5]]]}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.37109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[61.5]]], "params": {"weight": [[0.4375]]}}, "language_model.model.layers.39.feed_forward.shared_expert.down_proj": {"inputs": [[[124.0]]], "outputs": [[[21.0]], [[2.007134273635223e+38]]], "params": {"weight": [[0.498046875]]}}, "language_model.model.layers.40.self_attn.qkv_proj": {"inputs": [[[46.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.40.self_attn.o_proj": {"inputs": [[[12.3125]]], "outputs": [[[93.0]], [[90.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.40.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.7890625]], [[13.1875]]]}, "language_model.model.layers.40.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[14.9375]]]}, "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.7890625]]]}, "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[11.375]]]}, "language_model.model.layers.40.self_attn.attn.impl.k_cache": {"inputs": [[[13.1875]]]}, "language_model.model.layers.40.self_attn.attn.impl.v_cache": {"inputs": [[[14.9375]]]}, "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[22.625]], [[13.1875]], [[13.8125]]], "outputs": [[[12.3125]], [[1.0]]]}, "language_model.model.layers.40.feed_forward.gate_up_proj": {"inputs": [[[40.75]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.40.feed_forward.down_proj": {"inputs": [[[164.0]]], "outputs": [[[22.625]], [[146.0]]], "params": {"weight": [[0.6328125]]}}, "language_model.model.layers.41.self_attn.qkv_proj": {"inputs": [[[33.75]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.41.self_attn.o_proj": {"inputs": [[[10.9375]]], "outputs": [[[159.0]], [[164.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.41.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.4921875]], [[18.0]]]}, "language_model.model.layers.41.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.6875]]]}, "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.4921875]]]}, "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[10.8125]]]}, "language_model.model.layers.41.self_attn.attn.impl.k_cache": {"inputs": [[[18.0]]]}, "language_model.model.layers.41.self_attn.attn.impl.v_cache": {"inputs": [[[12.6875]]]}, "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[20.25]], [[18.0]], [[12.6875]]], "outputs": [[[10.9375]], [[1.0]]]}, "language_model.model.layers.41.feed_forward.router": {"inputs": [[[68.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op": {"inputs": [[[68.0]]], "outputs": [[[20.0]], [[80.5]], [[40.75]], [[29.75]], [[87.0]], [[39.5]], [[43.0]], [[129.0]], [[54.25]], [[79.5]], [[103.0]], [[176.0]], [[68.5]], [[78.5]], [[127.5]], [[105.0]], [[170.0]], [[98.0]], [[82.0]], [[169.0]], [[37.75]], [[76.0]], [[39.5]], [[125.5]], [[82.5]], [[161.0]], [[156.0]], [[100.5]], [[54.25]], [[72.0]], [[104.0]], [[53.0]], [[70.5]], [[172.0]], [[66.0]], [[106.0]], [[129.0]], [[58.25]], [[119.0]], [[42.5]], [[64.5]], [[44.5]], [[129.0]], [[107.0]], [[40.0]], [[75.0]], [[129.0]], [[63.75]], [[85.5]], [[220.0]], [[102.5]], [[101.5]], [[72.5]], [[85.5]], [[112.5]], [[101.0]], [[100.0]], [[154.0]], [[55.0]], [[81.5]], [[69.0]], [[120.5]], [[86.0]], [[58.0]], [[76.5]], [[117.0]], [[49.75]], [[85.0]], [[85.5]], [[44.25]], [[103.0]], [[55.25]], [[58.25]], [[65.5]], [[27.25]], [[102.5]], [[156.0]], [[77.5]], [[49.0]], [[54.25]], [[105.5]], [[133.0]], [[162.0]], [[97.5]], [[57.5]], [[44.0]], [[41.25]], [[71.0]], [[67.0]], [[59.5]], [[70.0]], [[85.0]], [[98.5]], [[158.0]], [[88.0]], [[58.75]], [[85.5]], [[120.0]], [[70.5]], [[76.5]], [[94.0]], [[115.0]], [[95.5]], [[88.0]], [[78.5]], [[71.5]], [[62.5]], [[72.5]], [[78.5]], [[29.0]], [[89.0]], [[183.0]], [[108.5]], [[89.0]], [[117.0]], [[50.5]], [[158.0]], [[91.5]], [[145.0]], [[131.0]], [[62.5]], [[61.75]], [[61.75]], [[111.0]], [[44.0]], [[51.5]], [[130.0]], [[65.5]], [[117.5]]]}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.39453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.4375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.361328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[68.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.41.feed_forward.shared_expert.down_proj": {"inputs": [[[219.0]]], "outputs": [[[56.25]], [[2.007134273635223e+38]]], "params": {"weight": [[1.0234375]]}}, "language_model.model.layers.42.self_attn.qkv_proj": {"inputs": [[[38.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.42.self_attn.o_proj": {"inputs": [[[13.0625]]], "outputs": [[[50.5]], [[134.0]]], "params": {"weight": [[0.5703125]]}}, "language_model.model.layers.42.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.2421875]], [[13.9375]]]}, "language_model.model.layers.42.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[15.5625]]]}, "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.2421875]]]}, "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[9.25]]]}, "language_model.model.layers.42.self_attn.attn.impl.k_cache": {"inputs": [[[13.9375]]]}, "language_model.model.layers.42.self_attn.attn.impl.v_cache": {"inputs": [[[15.5625]]]}, "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.75]], [[13.9375]], [[15.5625]]], "outputs": [[[13.0625]], [[1.0]]]}, "language_model.model.layers.42.feed_forward.gate_up_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.42.feed_forward.down_proj": {"inputs": [[[161.0]]], "outputs": [[[59.5]], [[338.0]]], "params": {"weight": [[0.71875]]}}, "language_model.model.layers.43.self_attn.qkv_proj": {"inputs": [[[45.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.43.self_attn.o_proj": {"inputs": [[[15.3125]]], "outputs": [[[94.5]], [[232.0]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.43.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.125]], [[13.1875]]]}, "language_model.model.layers.43.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[17.0]]]}, "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.125]]]}, "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[15.3125]]]}, "language_model.model.layers.43.self_attn.attn.impl.k_cache": {"inputs": [[[13.1875]]]}, "language_model.model.layers.43.self_attn.attn.impl.v_cache": {"inputs": [[[17.0]]]}, "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.125]], [[13.1875]], [[17.0]]], "outputs": [[[14.625]], [[1.0]]]}, "language_model.model.layers.43.feed_forward.router": {"inputs": [[[68.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op": {"inputs": [[[68.0]]], "outputs": [[[20.75]], [[48.5]], [[75.0]], [[62.5]], [[117.0]], [[154.0]], [[80.5]], [[48.75]], [[109.5]], [[65.5]], [[99.5]], [[87.5]], [[95.0]], [[107.5]], [[85.0]], [[60.5]], [[138.0]], [[62.75]], [[91.5]], [[184.0]], [[40.5]], [[225.0]], [[118.0]], [[113.0]], [[91.5]], [[102.0]], [[100.0]], [[114.5]], [[101.5]], [[99.5]], [[117.0]], [[60.0]], [[94.0]], [[61.5]], [[125.0]], [[89.5]], [[109.5]], [[138.0]], [[106.0]], [[76.5]], [[127.0]], [[78.5]], [[152.0]], [[167.0]], [[44.75]], [[210.0]], [[81.0]], [[86.0]], [[37.75]], [[116.5]], [[127.0]], [[43.5]], [[150.0]], [[122.0]], [[69.0]], [[112.0]], [[72.0]], [[38.25]], [[36.5]], [[81.0]], [[121.0]], [[48.75]], [[81.5]], [[63.0]], [[148.0]], [[169.0]], [[121.0]], [[115.0]], [[69.0]], [[147.0]], [[51.0]], [[68.0]], [[92.0]], [[83.0]], [[73.5]], [[57.0]], [[83.5]], [[161.0]], [[135.0]], [[166.0]], [[89.5]], [[136.0]], [[75.0]], [[98.0]], [[74.0]], [[94.0]], [[157.0]], [[69.5]], [[71.5]], [[88.5]], [[49.25]], [[117.0]], [[54.75]], [[142.0]], [[72.0]], [[49.0]], [[44.25]], [[36.75]], [[236.0]], [[39.75]], [[96.0]], [[77.0]], [[89.5]], [[145.0]], [[120.5]], [[101.0]], [[85.0]], [[75.0]], [[89.0]], [[57.25]], [[95.5]], [[180.0]], [[134.0]], [[110.5]], [[69.5]], [[37.75]], [[128.0]], [[71.0]], [[103.0]], [[107.0]], [[56.75]], [[52.75]], [[105.5]], [[96.5]], [[102.5]], [[86.5]], [[203.0]], [[70.0]], [[68.5]]]}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.49609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[68.0]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.43.feed_forward.shared_expert.down_proj": {"inputs": [[[230.0]]], "outputs": [[[35.25]], [[2.007134273635223e+38]]], "params": {"weight": [[0.59375]]}}, "language_model.model.layers.44.self_attn.qkv_proj": {"inputs": [[[28.625]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.44.self_attn.o_proj": {"inputs": [[[15.5625]]], "outputs": [[[68.0]], [[153.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.44.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.6796875]], [[25.5]]]}, "language_model.model.layers.44.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[18.125]]]}, "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.6796875]]]}, "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[10.1875]]]}, "language_model.model.layers.44.self_attn.attn.impl.k_cache": {"inputs": [[[25.5]]]}, "language_model.model.layers.44.self_attn.attn.impl.v_cache": {"inputs": [[[18.125]]]}, "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[20.125]], [[25.5]], [[18.125]]], "outputs": [[[15.5625]], [[1.0]]]}, "language_model.model.layers.44.feed_forward.gate_up_proj": {"inputs": [[[41.5]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.44.feed_forward.down_proj": {"inputs": [[[232.0]]], "outputs": [[[63.25]], [[191.0]]], "params": {"weight": [[0.51171875]]}}, "language_model.model.layers.45.self_attn.qkv_proj": {"inputs": [[[27.375]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.45.self_attn.o_proj": {"inputs": [[[16.75]]], "outputs": [[[169.0]], [[220.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.45.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.609375]], [[16.25]]]}, "language_model.model.layers.45.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[17.375]]]}, "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.609375]]]}, "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[12.6875]]]}, "language_model.model.layers.45.self_attn.attn.impl.k_cache": {"inputs": [[[16.25]]]}, "language_model.model.layers.45.self_attn.attn.impl.v_cache": {"inputs": [[[17.375]]]}, "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[19.25]], [[16.25]], [[17.375]]], "outputs": [[[16.75]], [[1.0]]]}, "language_model.model.layers.45.feed_forward.router": {"inputs": [[[51.75]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op": {"inputs": [[[51.75]]], "outputs": [[[27.625]], [[53.75]], [[16.625]], [[32.5]], [[61.5]], [[0.0]], [[38.25]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[32.75]], [[29.0]], [[27.875]], [[44.25]], [[0.0]], [[0.0]], [[0.0]], [[61.75]], [[0.0]], [[0.0]], [[0.0]], [[117.0]], [[0.0]], [[21.375]], [[31.0]], [[30.5]], [[67.5]], [[41.0]], [[0.0]], [[17.75]], [[17.25]], [[37.0]], [[51.5]], [[102.5]], [[54.0]], [[0.0]], [[35.75]], [[15.875]], [[48.25]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[97.5]], [[0.0]], [[65.0]], [[0.0]], [[0.0]], [[0.0]], [[66.0]], [[0.0]], [[199.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[56.25]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[245.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[61.5]], [[76.0]], [[62.75]], [[94.5]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[40.5]], [[0.0]], [[83.5]], [[31.375]], [[57.5]], [[56.25]], [[0.0]], [[0.0]], [[20.75]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[16.375]], [[7.589415207398531e-18]], [[53.75]], [[0.0]], [[61.0]], [[0.0]], [[0.0]]]}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.419921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[51.75]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.45.feed_forward.shared_expert.down_proj": {"inputs": [[[213.0]]], "outputs": [[[41.0]], [[2.007134273635223e+38]]], "params": {"weight": [[0.44921875]]}}, "language_model.model.layers.46.self_attn.qkv_proj": {"inputs": [[[26.75]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.46.self_attn.o_proj": {"inputs": [[[17.0]]], "outputs": [[[101.0]], [[188.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.46.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.6796875]], [[15.875]]]}, "language_model.model.layers.46.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[20.75]]]}, "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.6796875]]]}, "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[16.75]]]}, "language_model.model.layers.46.self_attn.attn.impl.k_cache": {"inputs": [[[15.875]]]}, "language_model.model.layers.46.self_attn.attn.impl.v_cache": {"inputs": [[[20.75]]]}, "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[19.125]], [[15.875]], [[20.75]]], "outputs": [[[17.0]], [[1.0]]]}, "language_model.model.layers.46.feed_forward.gate_up_proj": {"inputs": [[[40.5]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.46.feed_forward.down_proj": {"inputs": [[[310.0]]], "outputs": [[[1688.0]], [[2400.0]]], "params": {"weight": [[0.447265625]]}}, "language_model.model.layers.47.self_attn.qkv_proj": {"inputs": [[[19.875]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.47.self_attn.o_proj": {"inputs": [[[12.75]]], "outputs": [[[98.0]], [[126.5]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.47.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.4609375]], [[16.125]]]}, "language_model.model.layers.47.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[13.4375]]]}, "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.4609375]]]}, "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[12.75]]]}, "language_model.model.layers.47.self_attn.attn.impl.k_cache": {"inputs": [[[16.125]]]}, "language_model.model.layers.47.self_attn.attn.impl.v_cache": {"inputs": [[[13.4375]]]}, "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[5.8125]], [[16.125]], [[12.625]]], "outputs": [[[12.5]], [[1.0]]]}, "language_model.model.layers.47.feed_forward.router": {"inputs": [[[33.5]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op": {"inputs": [[[33.5]]], "outputs": [[[274.0]], [[74.0]], [[76.5]], [[47.5]], [[348.0]], [[77.0]], [[34.5]], [[58.75]], [[38.0]], [[139.0]], [[251.0]], [[79.0]], [[90.0]], [[134.0]], [[66.0]], [[100.0]], [[30.875]], [[67.0]], [[43.75]], [[47.75]], [[86.0]], [[60.75]], [[79.5]], [[50.75]], [[55.0]], [[247.0]], [[256.0]], [[26.375]], [[103.5]], [[47.5]], [[64.5]], [[26.875]], [[49.5]], [[40.5]], [[56.75]], [[137.0]], [[32.75]], [[29.375]], [[24.25]], [[282.0]], [[72.0]], [[45.0]], [[109.0]], [[168.0]], [[28.625]], [[56.0]], [[88.5]], [[223.0]], [[45.75]], [[21.625]], [[100.0]], [[88.0]], [[300.0]], [[61.5]], [[23.375]], [[54.0]], [[31.625]], [[152.0]], [[75.0]], [[35.5]], [[111.5]], [[107.5]], [[79.0]], [[164.0]], [[155.0]], [[41.5]], [[62.25]], [[39.5]], [[117.0]], [[26.125]], [[151.0]], [[231.0]], [[28.875]], [[25.75]], [[26.75]], [[47.25]], [[30.375]], [[42.75]], [[53.25]], [[154.0]], [[27.0]], [[122.5]], [[215.0]], [[520.0]], [[82.0]], [[30.25]], [[63.0]], [[142.0]], [[39.5]], [[35.75]], [[328.0]], [[170.0]], [[138.0]], [[227.0]], [[33.25]], [[39.75]], [[150.0]], [[71.5]], [[40.25]], [[25.125]], [[44.0]], [[97.0]], [[50.5]], [[69.5]], [[39.0]], [[75.0]], [[31.75]], [[71.0]], [[282.0]], [[67.5]], [[76.5]], [[129.0]], [[177.0]], [[47.25]], [[51.0]], [[213.0]], [[63.5]], [[77.5]], [[214.0]], [[43.25]], [[336.0]], [[142.0]], [[60.0]], [[73.0]], [[25.25]], [[32.25]], [[132.0]], [[236.0]], [[346.0]]]}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.5]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.67578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.4453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.5703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.75390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.34765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.6328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.3671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.80078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.7265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.84765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.4140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.3984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.67578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.38671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.392578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.59375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.55859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.6484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.5]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.60546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.65625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.51171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.50390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.69921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.90625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.7734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.83203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.75]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.82421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.5]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.48046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.458984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.58984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.5859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.341796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.4140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.5]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.63671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.46875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.4921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.48828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.376953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.3671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.55859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.6015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.462890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.51953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.59765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.78515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.380859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.72265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.7421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.7890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.75390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.5]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.5078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.73046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.7265625]]}}, "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[33.5]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.47.feed_forward.shared_expert.down_proj": {"inputs": [[[916.0]]], "outputs": [[[624.0]], [[2.007134273635223e+38]]], "params": {"weight": [[0.435546875]]}}, "language_model.lm_head": {"inputs": [[[52.0]]], "params": {"weight": [[1.0078125]]}}}} \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_3_8.npz b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_3_8.npz new file mode 100644 index 000000000000..683a8727f88f Binary files /dev/null and b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_3_8.npz differ diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_3_8_mod_list.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_3_8_mod_list.json new file mode 100644 index 000000000000..928cd9e2a2c5 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_3_8_mod_list.json @@ -0,0 +1,6935 @@ +[ + "vision_model.patch_embedding.linear", + "vision_model.model.layers.0.self_attn.qkv_proj", + "vision_model.model.layers.0.self_attn.o_proj", + "vision_model.model.layers.0.mlp.fc1", + "vision_model.model.layers.0.mlp.fc2", + "vision_model.model.layers.1.self_attn.qkv_proj", + "vision_model.model.layers.1.self_attn.o_proj", + "vision_model.model.layers.1.mlp.fc1", + "vision_model.model.layers.1.mlp.fc2", + "vision_model.model.layers.2.self_attn.qkv_proj", + "vision_model.model.layers.2.self_attn.o_proj", + "vision_model.model.layers.2.mlp.fc1", + "vision_model.model.layers.2.mlp.fc2", + "vision_model.model.layers.3.self_attn.qkv_proj", + "vision_model.model.layers.3.self_attn.o_proj", + "vision_model.model.layers.3.mlp.fc1", + "vision_model.model.layers.3.mlp.fc2", + "vision_model.model.layers.4.self_attn.qkv_proj", + "vision_model.model.layers.4.self_attn.o_proj", + "vision_model.model.layers.4.mlp.fc1", + "vision_model.model.layers.4.mlp.fc2", + "vision_model.model.layers.5.self_attn.qkv_proj", + "vision_model.model.layers.5.self_attn.o_proj", + "vision_model.model.layers.5.mlp.fc1", + "vision_model.model.layers.5.mlp.fc2", + "vision_model.model.layers.6.self_attn.qkv_proj", + "vision_model.model.layers.6.self_attn.o_proj", + "vision_model.model.layers.6.mlp.fc1", + "vision_model.model.layers.6.mlp.fc2", + "vision_model.model.layers.7.self_attn.qkv_proj", + "vision_model.model.layers.7.self_attn.o_proj", + "vision_model.model.layers.7.mlp.fc1", + "vision_model.model.layers.7.mlp.fc2", + "vision_model.model.layers.8.self_attn.qkv_proj", + "vision_model.model.layers.8.self_attn.o_proj", + "vision_model.model.layers.8.mlp.fc1", + "vision_model.model.layers.8.mlp.fc2", + "vision_model.model.layers.9.self_attn.qkv_proj", + "vision_model.model.layers.9.self_attn.o_proj", + "vision_model.model.layers.9.mlp.fc1", + "vision_model.model.layers.9.mlp.fc2", + "vision_model.model.layers.10.self_attn.qkv_proj", + "vision_model.model.layers.10.self_attn.o_proj", + "vision_model.model.layers.10.mlp.fc1", + "vision_model.model.layers.10.mlp.fc2", + "vision_model.model.layers.11.self_attn.qkv_proj", + "vision_model.model.layers.11.self_attn.o_proj", + "vision_model.model.layers.11.mlp.fc1", + "vision_model.model.layers.11.mlp.fc2", + "vision_model.model.layers.12.self_attn.qkv_proj", + "vision_model.model.layers.12.self_attn.o_proj", + "vision_model.model.layers.12.mlp.fc1", + "vision_model.model.layers.12.mlp.fc2", + "vision_model.model.layers.13.self_attn.qkv_proj", + "vision_model.model.layers.13.self_attn.o_proj", + "vision_model.model.layers.13.mlp.fc1", + "vision_model.model.layers.13.mlp.fc2", + "vision_model.model.layers.14.self_attn.qkv_proj", + "vision_model.model.layers.14.self_attn.o_proj", + "vision_model.model.layers.14.mlp.fc1", + "vision_model.model.layers.14.mlp.fc2", + "vision_model.model.layers.15.self_attn.qkv_proj", + "vision_model.model.layers.15.self_attn.o_proj", + "vision_model.model.layers.15.mlp.fc1", + "vision_model.model.layers.15.mlp.fc2", + "vision_model.model.layers.16.self_attn.qkv_proj", + "vision_model.model.layers.16.self_attn.o_proj", + "vision_model.model.layers.16.mlp.fc1", + "vision_model.model.layers.16.mlp.fc2", + "vision_model.model.layers.17.self_attn.qkv_proj", + "vision_model.model.layers.17.self_attn.o_proj", + "vision_model.model.layers.17.mlp.fc1", + "vision_model.model.layers.17.mlp.fc2", + "vision_model.model.layers.18.self_attn.qkv_proj", + "vision_model.model.layers.18.self_attn.o_proj", + "vision_model.model.layers.18.mlp.fc1", + "vision_model.model.layers.18.mlp.fc2", + "vision_model.model.layers.19.self_attn.qkv_proj", + "vision_model.model.layers.19.self_attn.o_proj", + "vision_model.model.layers.19.mlp.fc1", + "vision_model.model.layers.19.mlp.fc2", + "vision_model.model.layers.20.self_attn.qkv_proj", + "vision_model.model.layers.20.self_attn.o_proj", + "vision_model.model.layers.20.mlp.fc1", + "vision_model.model.layers.20.mlp.fc2", + "vision_model.model.layers.21.self_attn.qkv_proj", + "vision_model.model.layers.21.self_attn.o_proj", + "vision_model.model.layers.21.mlp.fc1", + "vision_model.model.layers.21.mlp.fc2", + "vision_model.model.layers.22.self_attn.qkv_proj", + "vision_model.model.layers.22.self_attn.o_proj", + "vision_model.model.layers.22.mlp.fc1", + "vision_model.model.layers.22.mlp.fc2", + "vision_model.model.layers.23.self_attn.qkv_proj", + "vision_model.model.layers.23.self_attn.o_proj", + "vision_model.model.layers.23.mlp.fc1", + "vision_model.model.layers.23.mlp.fc2", + "vision_model.model.layers.24.self_attn.qkv_proj", + "vision_model.model.layers.24.self_attn.o_proj", + "vision_model.model.layers.24.mlp.fc1", + "vision_model.model.layers.24.mlp.fc2", + "vision_model.model.layers.25.self_attn.qkv_proj", + "vision_model.model.layers.25.self_attn.o_proj", + "vision_model.model.layers.25.mlp.fc1", + "vision_model.model.layers.25.mlp.fc2", + "vision_model.model.layers.26.self_attn.qkv_proj", + "vision_model.model.layers.26.self_attn.o_proj", + "vision_model.model.layers.26.mlp.fc1", + "vision_model.model.layers.26.mlp.fc2", + "vision_model.model.layers.27.self_attn.qkv_proj", + "vision_model.model.layers.27.self_attn.o_proj", + "vision_model.model.layers.27.mlp.fc1", + "vision_model.model.layers.27.mlp.fc2", + "vision_model.model.layers.28.self_attn.qkv_proj", + "vision_model.model.layers.28.self_attn.o_proj", + "vision_model.model.layers.28.mlp.fc1", + "vision_model.model.layers.28.mlp.fc2", + "vision_model.model.layers.29.self_attn.qkv_proj", + "vision_model.model.layers.29.self_attn.o_proj", + "vision_model.model.layers.29.mlp.fc1", + "vision_model.model.layers.29.mlp.fc2", + "vision_model.model.layers.30.self_attn.qkv_proj", + "vision_model.model.layers.30.self_attn.o_proj", + "vision_model.model.layers.30.mlp.fc1", + "vision_model.model.layers.30.mlp.fc2", + "vision_model.model.layers.31.self_attn.qkv_proj", + "vision_model.model.layers.31.self_attn.o_proj", + "vision_model.model.layers.31.mlp.fc1", + "vision_model.model.layers.31.mlp.fc2", + "vision_model.model.layers.32.self_attn.qkv_proj", + "vision_model.model.layers.32.self_attn.o_proj", + "vision_model.model.layers.32.mlp.fc1", + "vision_model.model.layers.32.mlp.fc2", + "vision_model.model.layers.33.self_attn.qkv_proj", + "vision_model.model.layers.33.self_attn.o_proj", + "vision_model.model.layers.33.mlp.fc1", + "vision_model.model.layers.33.mlp.fc2", + "vision_model.vision_adapter.mlp.fc1", + "vision_model.vision_adapter.mlp.fc2", + "multi_modal_projector.linear_1", + "language_model.model.layers.0.self_attn.qkv_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.0.self_attn.attn.impl.softmax", + "language_model.model.layers.0.self_attn.attn.impl.matmul_av", + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.0.self_attn.attn.impl.k_cache", + "language_model.model.layers.0.self_attn.attn.impl.v_cache", + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.0.feed_forward.gate_up_proj", + "language_model.model.layers.0.feed_forward.down_proj", + "language_model.model.layers.1.self_attn.qkv_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.1.self_attn.attn.impl.softmax", + "language_model.model.layers.1.self_attn.attn.impl.matmul_av", + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.1.self_attn.attn.impl.k_cache", + "language_model.model.layers.1.self_attn.attn.impl.v_cache", + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.1.feed_forward.router", + "language_model.model.layers.1.feed_forward.experts", + "language_model.model.layers.1.feed_forward.experts.moe_op", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.1.feed_forward.shared_expert.down_proj", + "language_model.model.layers.2.self_attn.qkv_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.2.self_attn.attn.impl.softmax", + "language_model.model.layers.2.self_attn.attn.impl.matmul_av", + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.2.self_attn.attn.impl.k_cache", + "language_model.model.layers.2.self_attn.attn.impl.v_cache", + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.2.feed_forward.gate_up_proj", + "language_model.model.layers.2.feed_forward.down_proj", + "language_model.model.layers.3.self_attn.qkv_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.3.self_attn.attn.impl.softmax", + "language_model.model.layers.3.self_attn.attn.impl.matmul_av", + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.3.self_attn.attn.impl.k_cache", + "language_model.model.layers.3.self_attn.attn.impl.v_cache", + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.3.feed_forward.router", + "language_model.model.layers.3.feed_forward.experts", + "language_model.model.layers.3.feed_forward.experts.moe_op", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.3.feed_forward.shared_expert.down_proj", + "language_model.model.layers.4.self_attn.qkv_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.4.self_attn.attn.impl.softmax", + "language_model.model.layers.4.self_attn.attn.impl.matmul_av", + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.4.self_attn.attn.impl.k_cache", + "language_model.model.layers.4.self_attn.attn.impl.v_cache", + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.4.feed_forward.gate_up_proj", + "language_model.model.layers.4.feed_forward.down_proj", + "language_model.model.layers.5.self_attn.qkv_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.5.self_attn.attn.impl.softmax", + "language_model.model.layers.5.self_attn.attn.impl.matmul_av", + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.5.self_attn.attn.impl.k_cache", + "language_model.model.layers.5.self_attn.attn.impl.v_cache", + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.5.feed_forward.router", + "language_model.model.layers.5.feed_forward.experts", + "language_model.model.layers.5.feed_forward.experts.moe_op", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.5.feed_forward.shared_expert.down_proj", + "language_model.model.layers.6.self_attn.qkv_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.6.self_attn.attn.impl.softmax", + "language_model.model.layers.6.self_attn.attn.impl.matmul_av", + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.6.self_attn.attn.impl.k_cache", + "language_model.model.layers.6.self_attn.attn.impl.v_cache", + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.6.feed_forward.gate_up_proj", + "language_model.model.layers.6.feed_forward.down_proj", + "language_model.model.layers.7.self_attn.qkv_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.7.self_attn.attn.impl.softmax", + "language_model.model.layers.7.self_attn.attn.impl.matmul_av", + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.7.self_attn.attn.impl.k_cache", + "language_model.model.layers.7.self_attn.attn.impl.v_cache", + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.7.feed_forward.router", + "language_model.model.layers.7.feed_forward.experts", + "language_model.model.layers.7.feed_forward.experts.moe_op", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.7.feed_forward.shared_expert.down_proj", + "language_model.model.layers.8.self_attn.qkv_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.8.self_attn.attn.impl.softmax", + "language_model.model.layers.8.self_attn.attn.impl.matmul_av", + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.8.self_attn.attn.impl.k_cache", + "language_model.model.layers.8.self_attn.attn.impl.v_cache", + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.8.feed_forward.gate_up_proj", + "language_model.model.layers.8.feed_forward.down_proj", + "language_model.model.layers.9.self_attn.qkv_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.9.self_attn.attn.impl.softmax", + "language_model.model.layers.9.self_attn.attn.impl.matmul_av", + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.9.self_attn.attn.impl.k_cache", + "language_model.model.layers.9.self_attn.attn.impl.v_cache", + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.9.feed_forward.router", + "language_model.model.layers.9.feed_forward.experts", + "language_model.model.layers.9.feed_forward.experts.moe_op", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.9.feed_forward.shared_expert.down_proj", + "language_model.model.layers.10.self_attn.qkv_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.10.self_attn.attn.impl.softmax", + "language_model.model.layers.10.self_attn.attn.impl.matmul_av", + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.10.self_attn.attn.impl.k_cache", + "language_model.model.layers.10.self_attn.attn.impl.v_cache", + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.10.feed_forward.gate_up_proj", + "language_model.model.layers.10.feed_forward.down_proj", + "language_model.model.layers.11.self_attn.qkv_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.11.self_attn.attn.impl.softmax", + "language_model.model.layers.11.self_attn.attn.impl.matmul_av", + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.11.self_attn.attn.impl.k_cache", + "language_model.model.layers.11.self_attn.attn.impl.v_cache", + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.11.feed_forward.router", + "language_model.model.layers.11.feed_forward.experts", + "language_model.model.layers.11.feed_forward.experts.moe_op", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.11.feed_forward.shared_expert.down_proj", + "language_model.model.layers.12.self_attn.qkv_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.12.self_attn.attn.impl.softmax", + "language_model.model.layers.12.self_attn.attn.impl.matmul_av", + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.12.self_attn.attn.impl.k_cache", + "language_model.model.layers.12.self_attn.attn.impl.v_cache", + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.12.feed_forward.gate_up_proj", + "language_model.model.layers.12.feed_forward.down_proj", + "language_model.model.layers.13.self_attn.qkv_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.13.self_attn.attn.impl.softmax", + "language_model.model.layers.13.self_attn.attn.impl.matmul_av", + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.13.self_attn.attn.impl.k_cache", + "language_model.model.layers.13.self_attn.attn.impl.v_cache", + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.13.feed_forward.router", + "language_model.model.layers.13.feed_forward.experts", + "language_model.model.layers.13.feed_forward.experts.moe_op", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.13.feed_forward.shared_expert.down_proj", + "language_model.model.layers.14.self_attn.qkv_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.14.self_attn.attn.impl.softmax", + "language_model.model.layers.14.self_attn.attn.impl.matmul_av", + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.14.self_attn.attn.impl.k_cache", + "language_model.model.layers.14.self_attn.attn.impl.v_cache", + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.14.feed_forward.gate_up_proj", + "language_model.model.layers.14.feed_forward.down_proj", + "language_model.model.layers.15.self_attn.qkv_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.15.self_attn.attn.impl.softmax", + "language_model.model.layers.15.self_attn.attn.impl.matmul_av", + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.15.self_attn.attn.impl.k_cache", + "language_model.model.layers.15.self_attn.attn.impl.v_cache", + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.15.feed_forward.router", + "language_model.model.layers.15.feed_forward.experts", + "language_model.model.layers.15.feed_forward.experts.moe_op", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.15.feed_forward.shared_expert.down_proj", + "language_model.model.layers.16.self_attn.qkv_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.16.self_attn.attn.impl.softmax", + "language_model.model.layers.16.self_attn.attn.impl.matmul_av", + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.16.self_attn.attn.impl.k_cache", + "language_model.model.layers.16.self_attn.attn.impl.v_cache", + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.16.feed_forward.gate_up_proj", + "language_model.model.layers.16.feed_forward.down_proj", + "language_model.model.layers.17.self_attn.qkv_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.17.self_attn.attn.impl.softmax", + "language_model.model.layers.17.self_attn.attn.impl.matmul_av", + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.17.self_attn.attn.impl.k_cache", + "language_model.model.layers.17.self_attn.attn.impl.v_cache", + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.17.feed_forward.router", + "language_model.model.layers.17.feed_forward.experts", + "language_model.model.layers.17.feed_forward.experts.moe_op", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.17.feed_forward.shared_expert.down_proj", + "language_model.model.layers.18.self_attn.qkv_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.18.self_attn.attn.impl.softmax", + "language_model.model.layers.18.self_attn.attn.impl.matmul_av", + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.18.self_attn.attn.impl.k_cache", + "language_model.model.layers.18.self_attn.attn.impl.v_cache", + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.18.feed_forward.gate_up_proj", + "language_model.model.layers.18.feed_forward.down_proj", + "language_model.model.layers.19.self_attn.qkv_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.19.self_attn.attn.impl.softmax", + "language_model.model.layers.19.self_attn.attn.impl.matmul_av", + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.19.self_attn.attn.impl.k_cache", + "language_model.model.layers.19.self_attn.attn.impl.v_cache", + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.19.feed_forward.router", + "language_model.model.layers.19.feed_forward.experts", + "language_model.model.layers.19.feed_forward.experts.moe_op", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.19.feed_forward.shared_expert.down_proj", + "language_model.model.layers.20.self_attn.qkv_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.20.self_attn.attn.impl.softmax", + "language_model.model.layers.20.self_attn.attn.impl.matmul_av", + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.20.self_attn.attn.impl.k_cache", + "language_model.model.layers.20.self_attn.attn.impl.v_cache", + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.20.feed_forward.gate_up_proj", + "language_model.model.layers.20.feed_forward.down_proj", + "language_model.model.layers.21.self_attn.qkv_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.21.self_attn.attn.impl.softmax", + "language_model.model.layers.21.self_attn.attn.impl.matmul_av", + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.21.self_attn.attn.impl.k_cache", + "language_model.model.layers.21.self_attn.attn.impl.v_cache", + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.21.feed_forward.router", + "language_model.model.layers.21.feed_forward.experts", + "language_model.model.layers.21.feed_forward.experts.moe_op", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.21.feed_forward.shared_expert.down_proj", + "language_model.model.layers.22.self_attn.qkv_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.22.self_attn.attn.impl.softmax", + "language_model.model.layers.22.self_attn.attn.impl.matmul_av", + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.22.self_attn.attn.impl.k_cache", + "language_model.model.layers.22.self_attn.attn.impl.v_cache", + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.22.feed_forward.gate_up_proj", + "language_model.model.layers.22.feed_forward.down_proj", + "language_model.model.layers.23.self_attn.qkv_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.23.self_attn.attn.impl.softmax", + "language_model.model.layers.23.self_attn.attn.impl.matmul_av", + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.23.self_attn.attn.impl.k_cache", + "language_model.model.layers.23.self_attn.attn.impl.v_cache", + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.23.feed_forward.router", + "language_model.model.layers.23.feed_forward.experts", + "language_model.model.layers.23.feed_forward.experts.moe_op", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.23.feed_forward.shared_expert.down_proj", + "language_model.model.layers.24.self_attn.qkv_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.24.self_attn.attn.impl.softmax", + "language_model.model.layers.24.self_attn.attn.impl.matmul_av", + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.24.self_attn.attn.impl.k_cache", + "language_model.model.layers.24.self_attn.attn.impl.v_cache", + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.24.feed_forward.gate_up_proj", + "language_model.model.layers.24.feed_forward.down_proj", + "language_model.model.layers.25.self_attn.qkv_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.25.self_attn.attn.impl.softmax", + "language_model.model.layers.25.self_attn.attn.impl.matmul_av", + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.25.self_attn.attn.impl.k_cache", + "language_model.model.layers.25.self_attn.attn.impl.v_cache", + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.25.feed_forward.router", + "language_model.model.layers.25.feed_forward.experts", + "language_model.model.layers.25.feed_forward.experts.moe_op", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.25.feed_forward.shared_expert.down_proj", + "language_model.model.layers.26.self_attn.qkv_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.26.self_attn.attn.impl.softmax", + "language_model.model.layers.26.self_attn.attn.impl.matmul_av", + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.26.self_attn.attn.impl.k_cache", + "language_model.model.layers.26.self_attn.attn.impl.v_cache", + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.26.feed_forward.gate_up_proj", + "language_model.model.layers.26.feed_forward.down_proj", + "language_model.model.layers.27.self_attn.qkv_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.27.self_attn.attn.impl.softmax", + "language_model.model.layers.27.self_attn.attn.impl.matmul_av", + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.27.self_attn.attn.impl.k_cache", + "language_model.model.layers.27.self_attn.attn.impl.v_cache", + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.27.feed_forward.router", + "language_model.model.layers.27.feed_forward.experts", + "language_model.model.layers.27.feed_forward.experts.moe_op", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.27.feed_forward.shared_expert.down_proj", + "language_model.model.layers.28.self_attn.qkv_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.28.self_attn.attn.impl.softmax", + "language_model.model.layers.28.self_attn.attn.impl.matmul_av", + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.28.self_attn.attn.impl.k_cache", + "language_model.model.layers.28.self_attn.attn.impl.v_cache", + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.28.feed_forward.gate_up_proj", + "language_model.model.layers.28.feed_forward.down_proj", + "language_model.model.layers.29.self_attn.qkv_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.29.self_attn.attn.impl.softmax", + "language_model.model.layers.29.self_attn.attn.impl.matmul_av", + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.29.self_attn.attn.impl.k_cache", + "language_model.model.layers.29.self_attn.attn.impl.v_cache", + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.29.feed_forward.router", + "language_model.model.layers.29.feed_forward.experts", + "language_model.model.layers.29.feed_forward.experts.moe_op", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.29.feed_forward.shared_expert.down_proj", + "language_model.model.layers.30.self_attn.qkv_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.30.self_attn.attn.impl.softmax", + "language_model.model.layers.30.self_attn.attn.impl.matmul_av", + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.30.self_attn.attn.impl.k_cache", + "language_model.model.layers.30.self_attn.attn.impl.v_cache", + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.30.feed_forward.gate_up_proj", + "language_model.model.layers.30.feed_forward.down_proj", + "language_model.model.layers.31.self_attn.qkv_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.31.self_attn.attn.impl.softmax", + "language_model.model.layers.31.self_attn.attn.impl.matmul_av", + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.31.self_attn.attn.impl.k_cache", + "language_model.model.layers.31.self_attn.attn.impl.v_cache", + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.31.feed_forward.router", + "language_model.model.layers.31.feed_forward.experts", + "language_model.model.layers.31.feed_forward.experts.moe_op", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.31.feed_forward.shared_expert.down_proj", + "language_model.model.layers.32.self_attn.qkv_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.32.self_attn.attn.impl.softmax", + "language_model.model.layers.32.self_attn.attn.impl.matmul_av", + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.32.self_attn.attn.impl.k_cache", + "language_model.model.layers.32.self_attn.attn.impl.v_cache", + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.32.feed_forward.gate_up_proj", + "language_model.model.layers.32.feed_forward.down_proj", + "language_model.model.layers.33.self_attn.qkv_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.33.self_attn.attn.impl.softmax", + "language_model.model.layers.33.self_attn.attn.impl.matmul_av", + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.33.self_attn.attn.impl.k_cache", + "language_model.model.layers.33.self_attn.attn.impl.v_cache", + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.33.feed_forward.router", + "language_model.model.layers.33.feed_forward.experts", + "language_model.model.layers.33.feed_forward.experts.moe_op", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.33.feed_forward.shared_expert.down_proj", + "language_model.model.layers.34.self_attn.qkv_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.34.self_attn.attn.impl.softmax", + "language_model.model.layers.34.self_attn.attn.impl.matmul_av", + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.34.self_attn.attn.impl.k_cache", + "language_model.model.layers.34.self_attn.attn.impl.v_cache", + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.34.feed_forward.gate_up_proj", + "language_model.model.layers.34.feed_forward.down_proj", + "language_model.model.layers.35.self_attn.qkv_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.35.self_attn.attn.impl.softmax", + "language_model.model.layers.35.self_attn.attn.impl.matmul_av", + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.35.self_attn.attn.impl.k_cache", + "language_model.model.layers.35.self_attn.attn.impl.v_cache", + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.35.feed_forward.router", + "language_model.model.layers.35.feed_forward.experts", + "language_model.model.layers.35.feed_forward.experts.moe_op", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.35.feed_forward.shared_expert.down_proj", + "language_model.model.layers.36.self_attn.qkv_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.36.self_attn.attn.impl.softmax", + "language_model.model.layers.36.self_attn.attn.impl.matmul_av", + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.36.self_attn.attn.impl.k_cache", + "language_model.model.layers.36.self_attn.attn.impl.v_cache", + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.36.feed_forward.gate_up_proj", + "language_model.model.layers.36.feed_forward.down_proj", + "language_model.model.layers.37.self_attn.qkv_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.37.self_attn.attn.impl.softmax", + "language_model.model.layers.37.self_attn.attn.impl.matmul_av", + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.37.self_attn.attn.impl.k_cache", + "language_model.model.layers.37.self_attn.attn.impl.v_cache", + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.37.feed_forward.router", + "language_model.model.layers.37.feed_forward.experts", + "language_model.model.layers.37.feed_forward.experts.moe_op", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.37.feed_forward.shared_expert.down_proj", + "language_model.model.layers.38.self_attn.qkv_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.38.self_attn.attn.impl.softmax", + "language_model.model.layers.38.self_attn.attn.impl.matmul_av", + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.38.self_attn.attn.impl.k_cache", + "language_model.model.layers.38.self_attn.attn.impl.v_cache", + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.38.feed_forward.gate_up_proj", + "language_model.model.layers.38.feed_forward.down_proj", + "language_model.model.layers.39.self_attn.qkv_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.39.self_attn.attn.impl.softmax", + "language_model.model.layers.39.self_attn.attn.impl.matmul_av", + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.39.self_attn.attn.impl.k_cache", + "language_model.model.layers.39.self_attn.attn.impl.v_cache", + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.39.feed_forward.router", + "language_model.model.layers.39.feed_forward.experts", + "language_model.model.layers.39.feed_forward.experts.moe_op", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.39.feed_forward.shared_expert.down_proj", + "language_model.model.layers.40.self_attn.qkv_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.40.self_attn.attn.impl.softmax", + "language_model.model.layers.40.self_attn.attn.impl.matmul_av", + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.40.self_attn.attn.impl.k_cache", + "language_model.model.layers.40.self_attn.attn.impl.v_cache", + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.40.feed_forward.gate_up_proj", + "language_model.model.layers.40.feed_forward.down_proj", + "language_model.model.layers.41.self_attn.qkv_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.41.self_attn.attn.impl.softmax", + "language_model.model.layers.41.self_attn.attn.impl.matmul_av", + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.41.self_attn.attn.impl.k_cache", + "language_model.model.layers.41.self_attn.attn.impl.v_cache", + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.41.feed_forward.router", + "language_model.model.layers.41.feed_forward.experts", + "language_model.model.layers.41.feed_forward.experts.moe_op", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.41.feed_forward.shared_expert.down_proj", + "language_model.model.layers.42.self_attn.qkv_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.42.self_attn.attn.impl.softmax", + "language_model.model.layers.42.self_attn.attn.impl.matmul_av", + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.42.self_attn.attn.impl.k_cache", + "language_model.model.layers.42.self_attn.attn.impl.v_cache", + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.42.feed_forward.gate_up_proj", + "language_model.model.layers.42.feed_forward.down_proj", + "language_model.model.layers.43.self_attn.qkv_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.43.self_attn.attn.impl.softmax", + "language_model.model.layers.43.self_attn.attn.impl.matmul_av", + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.43.self_attn.attn.impl.k_cache", + "language_model.model.layers.43.self_attn.attn.impl.v_cache", + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.43.feed_forward.router", + "language_model.model.layers.43.feed_forward.experts", + "language_model.model.layers.43.feed_forward.experts.moe_op", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.43.feed_forward.shared_expert.down_proj", + "language_model.model.layers.44.self_attn.qkv_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.44.self_attn.attn.impl.softmax", + "language_model.model.layers.44.self_attn.attn.impl.matmul_av", + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.44.self_attn.attn.impl.k_cache", + "language_model.model.layers.44.self_attn.attn.impl.v_cache", + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.44.feed_forward.gate_up_proj", + "language_model.model.layers.44.feed_forward.down_proj", + "language_model.model.layers.45.self_attn.qkv_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.45.self_attn.attn.impl.softmax", + "language_model.model.layers.45.self_attn.attn.impl.matmul_av", + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.45.self_attn.attn.impl.k_cache", + "language_model.model.layers.45.self_attn.attn.impl.v_cache", + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.45.feed_forward.router", + "language_model.model.layers.45.feed_forward.experts", + "language_model.model.layers.45.feed_forward.experts.moe_op", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.45.feed_forward.shared_expert.down_proj", + "language_model.model.layers.46.self_attn.qkv_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.46.self_attn.attn.impl.softmax", + "language_model.model.layers.46.self_attn.attn.impl.matmul_av", + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.46.self_attn.attn.impl.k_cache", + "language_model.model.layers.46.self_attn.attn.impl.v_cache", + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.46.feed_forward.gate_up_proj", + "language_model.model.layers.46.feed_forward.down_proj", + "language_model.model.layers.47.self_attn.qkv_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.47.self_attn.attn.impl.softmax", + "language_model.model.layers.47.self_attn.attn.impl.matmul_av", + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.47.self_attn.attn.impl.k_cache", + "language_model.model.layers.47.self_attn.attn.impl.v_cache", + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.47.feed_forward.router", + "language_model.model.layers.47.feed_forward.experts", + "language_model.model.layers.47.feed_forward.experts.moe_op", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.47.feed_forward.shared_expert.down_proj", + "language_model.lm_head" +] \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_4_8.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_4_8.json new file mode 100644 index 000000000000..c00851007b67 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_4_8.json @@ -0,0 +1 @@ +{"GlobalRank": null, "LocalRank": 4, "Mode": "DynamicRange", "Nodes": {"language_model.model.layers.0.self_attn.qkv_proj": {"inputs": [[[11.1875]]], "params": {"weight": [[1.921875]]}}, "language_model.model.layers.0.self_attn.o_proj": {"inputs": [[[0.74609375]]], "outputs": [[[0.80078125]], [[3.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.0.self_attn.attn.impl.matmul_qk": {"inputs": [[[2.0625]], [[10.5]]]}, "language_model.model.layers.0.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.8046875]]]}, "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[2.0625]]]}, "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.8046875]]]}, "language_model.model.layers.0.self_attn.attn.impl.k_cache": {"inputs": [[[10.5]]]}, "language_model.model.layers.0.self_attn.attn.impl.v_cache": {"inputs": [[[0.8046875]]]}, "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[28.375]], [[10.5]], [[0.8046875]]], "outputs": [[[0.74609375]], [[1.0]]]}, "language_model.model.layers.0.feed_forward.gate_up_proj": {"inputs": [[[2.546875]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.0.feed_forward.down_proj": {"inputs": [[[3.578125]]], "outputs": [[[26.75]], [[65.0]]], "params": {"weight": [[0.578125]]}}, "language_model.model.layers.1.self_attn.qkv_proj": {"inputs": [[[12.3125]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.1.self_attn.o_proj": {"inputs": [[[0.5078125]]], "outputs": [[[0.52734375]], [[1.0078125]]], "params": {"weight": [[0.361328125]]}}, "language_model.model.layers.1.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.6796875]], [[9.5]]]}, "language_model.model.layers.1.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.7734375]]]}, "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.6796875]]]}, "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.3984375]]]}, "language_model.model.layers.1.self_attn.attn.impl.k_cache": {"inputs": [[[9.5]]]}, "language_model.model.layers.1.self_attn.attn.impl.v_cache": {"inputs": [[[1.7734375]]]}, "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[7.03125]], [[8.1875]], [[1.7734375]]], "outputs": [[[0.494140625]], [[1.0]]]}, "language_model.model.layers.1.feed_forward.router": {"inputs": [[[1.8828125]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op": {"inputs": [[[1.8828125]]], "outputs": [[[0.97265625]], [[6.5625]], [[0.0]], [[1.1640625]], [[12.75]], [[14.0]], [[10.5625]], [[13.0]], [[6.15625]], [[8.0]], [[13.5625]], [[10.875]], [[11.5]], [[3.0]], [[8.375]], [[9.875]], [[18.0]], [[4.21875]], [[2.765625]], [[10.4375]], [[4.125]], [[11.0625]], [[13.25]], [[0.0]], [[15.4375]], [[8.125]], [[1.0234375]], [[12.0625]], [[8.9375]], [[10.5625]], [[10.8125]], [[20.25]], [[13.875]], [[0.2197265625]], [[5.125]], [[0.0]], [[10.6875]], [[9.625]], [[10.4375]], [[3.140625]], [[19.625]], [[16.125]], [[12.75]], [[14.375]], [[10.6875]], [[3.046875]], [[11.875]], [[8.875]], [[7.8125]], [[3.0625]], [[0.0]], [[8.6875]], [[12.375]], [[6.96875]], [[4.28125]], [[15.8125]], [[13.9375]], [[10.25]], [[0.435546875]], [[10.6875]], [[7.53125]], [[11.1875]], [[12.75]], [[14.625]], [[0.0]], [[14.5]], [[6.9375]], [[5.84375]], [[10.5]], [[7.0625]], [[17.0]], [[12.9375]], [[12.0625]], [[14.375]], [[18.0]], [[10.5]], [[8.375]], [[10.25]], [[10.0625]], [[6.75]], [[12.25]], [[0.0]], [[2.53125]], [[9.875]], [[3.375]], [[15.875]], [[11.3125]], [[8.5]], [[11.375]], [[11.4375]], [[9.125]], [[11.875]], [[8.875]], [[11.6875]], [[0.0]], [[11.5625]], [[10.6875]], [[0.1171875]], [[0.0]], [[11.4375]], [[12.3125]], [[8.875]], [[9.125]], [[11.5625]], [[4.0]], [[5.53125]], [[8.6875]], [[0.057861328125]], [[0.6328125]], [[3.3125]], [[0.2421875]], [[11.25]], [[0.61328125]], [[14.875]], [[0.0]], [[17.375]], [[12.0625]], [[12.375]], [[6.0]], [[13.0]], [[13.4375]], [[9.625]], [[9.4375]], [[2.921875]], [[9.25]], [[6.90625]], [[6.3125]], [[9.9375]], [[15.25]]]}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0751953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.458984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.34765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.07373046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0712890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.07421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.5078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.357421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[1.8828125]]], "params": {"weight": [[0.3046875]]}}, "language_model.model.layers.1.feed_forward.shared_expert.down_proj": {"inputs": [[[5.34375]]], "outputs": [[[0.35546875]], [[9.902748568597623e+37]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.2.self_attn.qkv_proj": {"inputs": [[[9.875]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.2.self_attn.o_proj": {"inputs": [[[0.72265625]]], "outputs": [[[0.6875]], [[1.328125]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.2.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.6328125]], [[12.1875]]]}, "language_model.model.layers.2.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.9140625]]]}, "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.6328125]]]}, "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.66796875]]]}, "language_model.model.layers.2.self_attn.attn.impl.k_cache": {"inputs": [[[12.1875]]]}, "language_model.model.layers.2.self_attn.attn.impl.v_cache": {"inputs": [[[0.9140625]]]}, "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.125]], [[12.1875]], [[0.9140625]]], "outputs": [[[0.72265625]], [[1.0]]]}, "language_model.model.layers.2.feed_forward.gate_up_proj": {"inputs": [[[6.125]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.2.feed_forward.down_proj": {"inputs": [[[6.90625]]], "outputs": [[[4.75]], [[25.25]]], "params": {"weight": [[0.5703125]]}}, "language_model.model.layers.3.self_attn.qkv_proj": {"inputs": [[[7.21875]]], "params": {"weight": [[0.4609375]]}}, "language_model.model.layers.3.self_attn.o_proj": {"inputs": [[[0.21484375]]], "outputs": [[[0.59765625]], [[2.40625]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.3.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.07421875]], [[3.671875]]]}, "language_model.model.layers.3.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.2431640625]]]}, "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.07421875]]]}, "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.1982421875]]]}, "language_model.model.layers.3.self_attn.attn.impl.k_cache": {"inputs": [[[3.671875]]]}, "language_model.model.layers.3.self_attn.attn.impl.v_cache": {"inputs": [[[0.2431640625]]]}, "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[0.91015625]], [[3.671875]], [[0.2431640625]]], "outputs": [[[0.21484375]], [[1.0]]]}, "language_model.model.layers.3.feed_forward.router": {"inputs": [[[8.75]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op": {"inputs": [[[8.75]]], "outputs": [[[266.0]], [[0.416015625]], [[8.6875]], [[19.75]], [[6.59375]], [[0.0]], [[11.3125]], [[15.3125]], [[13.0]], [[13.1875]], [[11.5]], [[14.25]], [[7.09375]], [[9.75]], [[10.8125]], [[10.9375]], [[15.375]], [[0.58203125]], [[5.9375]], [[7.53125]], [[15.4375]], [[10.625]], [[20.75]], [[14.125]], [[15.375]], [[11.8125]], [[3.5625]], [[9.1875]], [[10.5625]], [[7.53125]], [[7.15625]], [[8.625]], [[6.0]], [[11.25]], [[14.625]], [[13.3125]], [[13.0625]], [[2.90625]], [[0.76171875]], [[17.625]], [[4.03125]], [[15.125]], [[12.3125]], [[3.765625]], [[14.5625]], [[17.0]], [[3.890625]], [[13.5]], [[13.75]], [[1.875]], [[14.5625]], [[14.125]], [[13.6875]], [[15.25]], [[1.1953125]], [[7.40625]], [[15.1875]], [[15.25]], [[8.75]], [[16.625]], [[7.65625]], [[3.125]], [[20.0]], [[16.5]], [[0.400390625]], [[10.4375]], [[13.125]], [[19.25]], [[0.1650390625]], [[6.0]], [[0.0]], [[14.25]], [[14.25]], [[22.875]], [[3.640625]], [[2.46875]], [[7.75]], [[14.8125]], [[0.142578125]], [[12.4375]], [[7.28125]], [[13.8125]], [[0.0]], [[4.4375]], [[171.0]], [[8.1875]], [[21.0]], [[17.0]], [[3.9375]], [[14.0625]], [[16.875]], [[12.25]], [[6.40625]], [[4.1875]], [[13.3125]], [[0.0]], [[10.5]], [[16.375]], [[11.1875]], [[13.0625]], [[1.0546875]], [[19.125]], [[1.390625]], [[4.96875]], [[11.6875]], [[0.0]], [[1.2109375]], [[9.375]], [[17.25]], [[6.375]], [[3.359375]], [[11.25]], [[9.3125]], [[12.3125]], [[2.0625]], [[5.375]], [[4.59375]], [[6.53125]], [[1.3828125]], [[4.96875]], [[6.0]], [[17.125]], [[7.21875]], [[6.125]], [[15.25]], [[2.859375]], [[14.0625]], [[3.875]], [[20.25]]]}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.7578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.07421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0751953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[8.75]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.3.feed_forward.shared_expert.down_proj": {"inputs": [[[516.0]]], "outputs": [[[452.0]], [[3.1502703500102506e+38]]], "params": {"weight": [[0.875]]}}, "language_model.model.layers.4.self_attn.qkv_proj": {"inputs": [[[13.75]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.4.self_attn.o_proj": {"inputs": [[[3.234375]]], "outputs": [[[1.015625]], [[2.046875]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.4.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.69921875]], [[7.78125]]]}, "language_model.model.layers.4.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.40625]]]}, "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.69921875]]]}, "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.984375]]]}, "language_model.model.layers.4.self_attn.attn.impl.k_cache": {"inputs": [[[7.78125]]]}, "language_model.model.layers.4.self_attn.attn.impl.v_cache": {"inputs": [[[5.40625]]]}, "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.6875]], [[7.78125]], [[5.40625]]], "outputs": [[[3.234375]], [[1.0]]]}, "language_model.model.layers.4.feed_forward.gate_up_proj": {"inputs": [[[7.09375]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.4.feed_forward.down_proj": {"inputs": [[[8.3125]]], "outputs": [[[2.796875]], [[4.96875]]], "params": {"weight": [[0.462890625]]}}, "language_model.model.layers.5.self_attn.qkv_proj": {"inputs": [[[17.375]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.5.self_attn.o_proj": {"inputs": [[[1.609375]]], "outputs": [[[0.97265625]], [[2.21875]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.5.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.75]], [[13.5625]]]}, "language_model.model.layers.5.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.140625]]]}, "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.75]]]}, "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.578125]]]}, "language_model.model.layers.5.self_attn.attn.impl.k_cache": {"inputs": [[[13.5625]]]}, "language_model.model.layers.5.self_attn.attn.impl.v_cache": {"inputs": [[[2.140625]]]}, "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.25]], [[13.5625]], [[2.140625]]], "outputs": [[[1.609375]], [[1.0]]]}, "language_model.model.layers.5.feed_forward.router": {"inputs": [[[19.25]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op": {"inputs": [[[19.25]]], "outputs": [[[179.0]], [[19.5]], [[13.4375]], [[11.875]], [[10.3125]], [[19.625]], [[5.90625]], [[4.8125]], [[13.875]], [[10.875]], [[26.375]], [[12.5]], [[18.5]], [[18.0]], [[3.640625]], [[8.6875]], [[12.625]], [[9.625]], [[10.0625]], [[23.125]], [[7.65625]], [[20.25]], [[19.625]], [[7.0625]], [[14.9375]], [[6.28125]], [[4.90625]], [[20.0]], [[196.0]], [[11.625]], [[24.375]], [[20.25]], [[7.4375]], [[11.9375]], [[27.875]], [[11.25]], [[12.3125]], [[21.875]], [[5.375]], [[11.375]], [[20.25]], [[10.1875]], [[4.1875]], [[4.28125]], [[14.125]], [[9.8125]], [[15.4375]], [[9.0625]], [[26.875]], [[3.4375]], [[13.75]], [[5.21875]], [[13.875]], [[1.0859375]], [[4.625]], [[10.75]], [[2.828125]], [[10.625]], [[21.625]], [[18.375]], [[2.9375]], [[7.25]], [[15.0625]], [[4.53125]], [[13.0]], [[5.59375]], [[11.4375]], [[5.96875]], [[12.0625]], [[14.375]], [[5.84375]], [[7.125]], [[19.5]], [[16.625]], [[15.5625]], [[7.125]], [[24.75]], [[22.25]], [[9.4375]], [[15.9375]], [[18.625]], [[2.453125]], [[12.3125]], [[10.125]], [[10.625]], [[4.84375]], [[10.8125]], [[0.224609375]], [[19.875]], [[14.75]], [[29.0]], [[25.25]], [[12.4375]], [[22.75]], [[17.5]], [[16.25]], [[6.78125]], [[14.6875]], [[10.125]], [[6.03125]], [[19.875]], [[13.0]], [[1.8515625]], [[2.3125]], [[27.25]], [[14.25]], [[6.9375]], [[5.1875]], [[13.75]], [[10.5625]], [[19.375]], [[9.875]], [[2.859375]], [[3.921875]], [[8.25]], [[29.5]], [[24.75]], [[2.59375]], [[13.4375]], [[5.8125]], [[13.0]], [[22.25]], [[12.9375]], [[25.625]], [[21.375]], [[15.375]], [[7.0]], [[5.03125]], [[3.703125]]]}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.388671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.78125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.59375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0751953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.6953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[19.25]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.5.feed_forward.shared_expert.down_proj": {"inputs": [[[6.09375]]], "outputs": [[[1.1640625]], [[3.1502703500102506e+38]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.6.self_attn.qkv_proj": {"inputs": [[[12.9375]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.6.self_attn.o_proj": {"inputs": [[[2.453125]]], "outputs": [[[0.66796875]], [[1.5234375]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.6.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.67578125]], [[13.6875]]]}, "language_model.model.layers.6.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.03125]]]}, "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.67578125]]]}, "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.359375]]]}, "language_model.model.layers.6.self_attn.attn.impl.k_cache": {"inputs": [[[13.6875]]]}, "language_model.model.layers.6.self_attn.attn.impl.v_cache": {"inputs": [[[4.03125]]]}, "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.75]], [[13.6875]], [[4.03125]]], "outputs": [[[2.453125]], [[1.0]]]}, "language_model.model.layers.6.feed_forward.gate_up_proj": {"inputs": [[[7.71875]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.6.feed_forward.down_proj": {"inputs": [[[7.0625]]], "outputs": [[[2.328125]], [[11.1875]]], "params": {"weight": [[0.37109375]]}}, "language_model.model.layers.7.self_attn.qkv_proj": {"inputs": [[[7.09375]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.7.self_attn.o_proj": {"inputs": [[[0.80078125]]], "outputs": [[[1.84375]], [[3.84375]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.7.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.380859375]], [[5.1875]]]}, "language_model.model.layers.7.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.171875]]]}, "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.380859375]]]}, "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.765625]]]}, "language_model.model.layers.7.self_attn.attn.impl.k_cache": {"inputs": [[[5.1875]]]}, "language_model.model.layers.7.self_attn.attn.impl.v_cache": {"inputs": [[[1.171875]]]}, "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[4.9375]], [[5.1875]], [[1.171875]]], "outputs": [[[0.80078125]], [[1.0]]]}, "language_model.model.layers.7.feed_forward.router": {"inputs": [[[11.6875]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op": {"inputs": [[[11.6875]]], "outputs": [[[2.75]], [[7.34375]], [[10.9375]], [[18.25]], [[20.5]], [[19.125]], [[16.25]], [[9.9375]], [[22.625]], [[10.4375]], [[14.5625]], [[20.625]], [[10.125]], [[8.4375]], [[23.875]], [[3.4375]], [[9.3125]], [[24.625]], [[11.6875]], [[21.125]], [[13.3125]], [[15.3125]], [[22.25]], [[29.0]], [[25.5]], [[20.0]], [[11.3125]], [[4.375]], [[5.1875]], [[17.125]], [[9.9375]], [[36.5]], [[25.0]], [[8.625]], [[7.84375]], [[7.0]], [[16.125]], [[6.03125]], [[16.375]], [[26.5]], [[1.90625]], [[1.546875]], [[4.5625]], [[1.21875]], [[16.375]], [[10.6875]], [[18.375]], [[0.5546875]], [[24.25]], [[21.5]], [[17.25]], [[23.0]], [[27.5]], [[29.625]], [[9.0625]], [[31.5]], [[28.875]], [[9.6875]], [[2.421875]], [[10.1875]], [[8.1875]], [[12.5]], [[29.375]], [[12.3125]], [[18.25]], [[20.75]], [[10.6875]], [[7.46875]], [[11.5]], [[8.4375]], [[1.8828125]], [[23.75]], [[3.125]], [[18.375]], [[8.4375]], [[38.25]], [[3.4375]], [[5.84375]], [[8.0]], [[1.609375]], [[9.5]], [[17.375]], [[20.875]], [[1.4140625]], [[6.71875]], [[3.359375]], [[11.875]], [[1.4921875]], [[14.625]], [[2.84375]], [[22.375]], [[11.9375]], [[22.5]], [[8.3125]], [[2.859375]], [[26.625]], [[5.3125]], [[16.0]], [[19.5]], [[21.5]], [[8.5625]], [[10.375]], [[16.25]], [[13.0625]], [[0.83203125]], [[26.25]], [[37.0]], [[17.625]], [[13.125]], [[0.9453125]], [[25.875]], [[12.6875]], [[21.5]], [[9.625]], [[26.125]], [[21.75]], [[9.0]], [[6.875]], [[18.375]], [[2.125]], [[8.8125]], [[24.25]], [[26.75]], [[17.5]], [[6.9375]], [[4.40625]], [[2.875]], [[22.875]], [[10.25]]]}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.435546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[11.6875]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.7.feed_forward.shared_expert.down_proj": {"inputs": [[[44.25]]], "outputs": [[[3.984375]], [[2.3053798051894635e+36]]], "params": {"weight": [[0.3046875]]}}, "language_model.model.layers.8.self_attn.qkv_proj": {"inputs": [[[15.125]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.8.self_attn.o_proj": {"inputs": [[[2.65625]]], "outputs": [[[1.9375]], [[2.109375]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.8.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.6640625]], [[9.5]]]}, "language_model.model.layers.8.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.046875]]]}, "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.6640625]]]}, "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.1875]]]}, "language_model.model.layers.8.self_attn.attn.impl.k_cache": {"inputs": [[[9.5]]]}, "language_model.model.layers.8.self_attn.attn.impl.v_cache": {"inputs": [[[3.046875]]]}, "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.4375]], [[9.5]], [[3.046875]]], "outputs": [[[2.65625]], [[1.0]]]}, "language_model.model.layers.8.feed_forward.gate_up_proj": {"inputs": [[[8.25]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.8.feed_forward.down_proj": {"inputs": [[[9.6875]]], "outputs": [[[2.5625]], [[9.25]]], "params": {"weight": [[0.55078125]]}}, "language_model.model.layers.9.self_attn.qkv_proj": {"inputs": [[[15.75]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.9.self_attn.o_proj": {"inputs": [[[2.296875]]], "outputs": [[[1.3125]], [[2.875]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.9.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.85546875]], [[17.0]]]}, "language_model.model.layers.9.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.015625]]]}, "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.85546875]]]}, "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.734375]]]}, "language_model.model.layers.9.self_attn.attn.impl.k_cache": {"inputs": [[[17.0]]]}, "language_model.model.layers.9.self_attn.attn.impl.v_cache": {"inputs": [[[3.015625]]]}, "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.9375]], [[17.0]], [[3.015625]]], "outputs": [[[2.296875]], [[1.0]]]}, "language_model.model.layers.9.feed_forward.router": {"inputs": [[[11.75]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op": {"inputs": [[[11.75]]], "outputs": [[[3.8125]], [[10.0]], [[21.75]], [[4.71875]], [[42.75]], [[30.75]], [[47.5]], [[19.5]], [[39.0]], [[27.75]], [[33.0]], [[23.0]], [[9.5625]], [[43.5]], [[28.25]], [[12.5625]], [[6.0625]], [[12.5625]], [[44.75]], [[41.5]], [[18.125]], [[16.0]], [[18.75]], [[5.59375]], [[20.0]], [[13.5]], [[17.75]], [[34.25]], [[20.25]], [[12.4375]], [[27.375]], [[18.75]], [[20.0]], [[28.625]], [[20.75]], [[43.75]], [[5.34375]], [[27.5]], [[15.6875]], [[14.5]], [[16.375]], [[5.75]], [[13.1875]], [[19.125]], [[33.5]], [[10.0625]], [[12.25]], [[15.6875]], [[20.625]], [[40.0]], [[23.75]], [[10.3125]], [[51.5]], [[20.0]], [[10.375]], [[17.0]], [[11.25]], [[31.75]], [[22.875]], [[33.5]], [[24.25]], [[19.875]], [[13.125]], [[22.125]], [[26.875]], [[11.25]], [[16.375]], [[50.0]], [[23.75]], [[5.96875]], [[22.125]], [[32.0]], [[34.5]], [[6.90625]], [[13.0]], [[10.0]], [[16.0]], [[36.5]], [[45.25]], [[9.1875]], [[23.625]], [[17.125]], [[30.75]], [[28.625]], [[28.0]], [[21.0]], [[41.75]], [[20.625]], [[31.625]], [[15.25]], [[40.5]], [[7.125]], [[9.5]], [[21.0]], [[15.375]], [[22.875]], [[23.0]], [[14.625]], [[5.625]], [[17.125]], [[17.375]], [[27.5]], [[13.0625]], [[38.75]], [[11.375]], [[33.25]], [[25.875]], [[34.75]], [[46.0]], [[45.75]], [[21.125]], [[18.875]], [[6.5]], [[23.75]], [[11.125]], [[28.5]], [[33.75]], [[8.8125]], [[5.375]], [[14.75]], [[13.6875]], [[23.0]], [[10.3125]], [[30.75]], [[13.75]], [[43.75]], [[19.875]], [[12.625]], [[24.5]]]}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.53515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[11.75]]], "params": {"weight": [[0.3046875]]}}, "language_model.model.layers.9.feed_forward.shared_expert.down_proj": {"inputs": [[[12.25]]], "outputs": [[[1.9140625]], [[3.1502703500102506e+38]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.10.self_attn.qkv_proj": {"inputs": [[[16.125]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.10.self_attn.o_proj": {"inputs": [[[2.671875]]], "outputs": [[[1.6015625]], [[3.09375]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.10.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.95703125]], [[17.25]]]}, "language_model.model.layers.10.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.046875]]]}, "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.95703125]]]}, "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.140625]]]}, "language_model.model.layers.10.self_attn.attn.impl.k_cache": {"inputs": [[[17.25]]]}, "language_model.model.layers.10.self_attn.attn.impl.v_cache": {"inputs": [[[3.046875]]]}, "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.25]], [[17.25]], [[3.046875]]], "outputs": [[[2.671875]], [[1.0]]]}, "language_model.model.layers.10.feed_forward.gate_up_proj": {"inputs": [[[9.875]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.10.feed_forward.down_proj": {"inputs": [[[19.875]]], "outputs": [[[6.90625]], [[11.3125]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.11.self_attn.qkv_proj": {"inputs": [[[13.5625]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.11.self_attn.o_proj": {"inputs": [[[4.1875]]], "outputs": [[[8.0625]], [[7.875]]], "params": {"weight": [[0.921875]]}}, "language_model.model.layers.11.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.70703125]], [[7.03125]]]}, "language_model.model.layers.11.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.71875]]]}, "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.70703125]]]}, "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.5625]]]}, "language_model.model.layers.11.self_attn.attn.impl.k_cache": {"inputs": [[[7.03125]]]}, "language_model.model.layers.11.self_attn.attn.impl.v_cache": {"inputs": [[[5.71875]]]}, "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.375]], [[7.03125]], [[5.71875]]], "outputs": [[[4.1875]], [[1.0]]]}, "language_model.model.layers.11.feed_forward.router": {"inputs": [[[13.375]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op": {"inputs": [[[13.375]]], "outputs": [[[3.671875]], [[24.25]], [[12.125]], [[17.375]], [[9.875]], [[18.875]], [[51.0]], [[1.8828125]], [[15.75]], [[22.0]], [[8.75]], [[32.0]], [[15.4375]], [[39.5]], [[37.0]], [[29.0]], [[57.5]], [[37.5]], [[40.25]], [[42.5]], [[17.375]], [[13.0]], [[23.375]], [[22.375]], [[14.75]], [[39.5]], [[41.75]], [[25.75]], [[15.3125]], [[10.625]], [[42.25]], [[31.25]], [[9.8125]], [[11.25]], [[30.75]], [[42.75]], [[28.375]], [[12.125]], [[19.875]], [[56.75]], [[21.625]], [[7.6875]], [[7.5625]], [[6.53125]], [[36.25]], [[33.75]], [[3.984375]], [[17.25]], [[35.75]], [[29.0]], [[24.0]], [[25.125]], [[17.5]], [[23.875]], [[21.625]], [[65.0]], [[48.5]], [[30.875]], [[7.75]], [[25.125]], [[21.25]], [[42.75]], [[27.75]], [[36.0]], [[25.5]], [[12.375]], [[27.75]], [[15.0625]], [[37.75]], [[14.375]], [[36.25]], [[9.8125]], [[8.6875]], [[1.8125]], [[21.625]], [[12.9375]], [[27.375]], [[16.375]], [[7.75]], [[33.5]], [[25.75]], [[16.75]], [[12.9375]], [[26.0]], [[7.15625]], [[2.046875]], [[1.609375]], [[49.75]], [[9.3125]], [[7.65625]], [[29.25]], [[49.5]], [[10.6875]], [[26.125]], [[58.0]], [[8.9375]], [[14.8125]], [[24.5]], [[30.125]], [[10.25]], [[19.375]], [[9.0625]], [[48.5]], [[16.75]], [[27.0]], [[42.75]], [[25.75]], [[20.125]], [[9.875]], [[16.0]], [[11.8125]], [[18.0]], [[6.21875]], [[2.09375]], [[18.125]], [[31.875]], [[42.75]], [[43.25]], [[63.5]], [[20.125]], [[29.5]], [[20.5]], [[15.6875]], [[21.0]], [[12.75]], [[6.375]], [[9.5]], [[32.75]], [[25.25]]]}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[13.375]]], "params": {"weight": [[0.3828125]]}}, "language_model.model.layers.11.feed_forward.shared_expert.down_proj": {"inputs": [[[12.4375]]], "outputs": [[[1.46875]], [[3.1502703500102506e+38]]], "params": {"weight": [[0.359375]]}}, "language_model.model.layers.12.self_attn.qkv_proj": {"inputs": [[[18.625]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.12.self_attn.o_proj": {"inputs": [[[2.8125]]], "outputs": [[[1.3359375]], [[3.640625]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.12.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.69921875]], [[12.6875]]]}, "language_model.model.layers.12.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.625]]]}, "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.69921875]]]}, "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.546875]]]}, "language_model.model.layers.12.self_attn.attn.impl.k_cache": {"inputs": [[[12.6875]]]}, "language_model.model.layers.12.self_attn.attn.impl.v_cache": {"inputs": [[[3.625]]]}, "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.4375]], [[12.3125]], [[3.625]]], "outputs": [[[2.8125]], [[1.0]]]}, "language_model.model.layers.12.feed_forward.gate_up_proj": {"inputs": [[[10.75]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.12.feed_forward.down_proj": {"inputs": [[[12.875]]], "outputs": [[[9.0]], [[17.375]]], "params": {"weight": [[0.67578125]]}}, "language_model.model.layers.13.self_attn.qkv_proj": {"inputs": [[[20.875]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.13.self_attn.o_proj": {"inputs": [[[3.046875]]], "outputs": [[[1.6875]], [[6.9375]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.13.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.82421875]], [[15.8125]]]}, "language_model.model.layers.13.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.09375]]]}, "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.82421875]]]}, "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.015625]]]}, "language_model.model.layers.13.self_attn.attn.impl.k_cache": {"inputs": [[[15.8125]]]}, "language_model.model.layers.13.self_attn.attn.impl.v_cache": {"inputs": [[[4.09375]]]}, "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.5625]], [[15.8125]], [[4.09375]]], "outputs": [[[3.046875]], [[1.0]]]}, "language_model.model.layers.13.feed_forward.router": {"inputs": [[[17.625]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op": {"inputs": [[[17.625]]], "outputs": [[[4.09375]], [[39.0]], [[43.0]], [[15.375]], [[43.5]], [[46.0]], [[35.0]], [[11.5]], [[14.25]], [[22.5]], [[2.71875]], [[28.875]], [[55.25]], [[23.875]], [[16.125]], [[60.75]], [[33.0]], [[20.25]], [[22.875]], [[14.0]], [[20.125]], [[23.5]], [[37.0]], [[6.0]], [[4.625]], [[10.4375]], [[47.0]], [[11.875]], [[62.25]], [[32.75]], [[14.9375]], [[19.125]], [[28.125]], [[59.25]], [[21.875]], [[53.75]], [[23.875]], [[56.5]], [[50.25]], [[16.0]], [[9.0]], [[25.25]], [[30.25]], [[31.25]], [[13.25]], [[26.125]], [[59.5]], [[30.75]], [[26.875]], [[48.25]], [[46.5]], [[26.125]], [[39.0]], [[19.5]], [[64.0]], [[4.71875]], [[22.125]], [[38.0]], [[37.75]], [[22.875]], [[29.625]], [[6.375]], [[43.5]], [[10.0625]], [[32.25]], [[18.0]], [[33.5]], [[18.25]], [[27.5]], [[31.875]], [[8.0]], [[55.75]], [[37.25]], [[8.6875]], [[15.125]], [[40.75]], [[41.0]], [[23.625]], [[30.625]], [[32.5]], [[24.875]], [[14.4375]], [[53.25]], [[3.140625]], [[73.0]], [[35.25]], [[33.25]], [[38.5]], [[18.875]], [[5.125]], [[59.75]], [[60.0]], [[64.0]], [[3.1875]], [[36.25]], [[43.25]], [[7.0625]], [[11.1875]], [[15.5]], [[22.0]], [[64.5]], [[12.25]], [[2.84375]], [[21.75]], [[35.75]], [[35.75]], [[25.625]], [[36.25]], [[24.75]], [[23.875]], [[31.125]], [[23.875]], [[32.5]], [[11.5625]], [[66.0]], [[18.0]], [[39.25]], [[27.5]], [[29.875]], [[41.5]], [[32.75]], [[28.0]], [[23.25]], [[28.25]], [[8.875]], [[40.5]], [[19.25]], [[6.375]], [[35.5]]]}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.5]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[17.625]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.13.feed_forward.shared_expert.down_proj": {"inputs": [[[16.625]]], "outputs": [[[3.421875]], [[3.1635626299680998e+38]]], "params": {"weight": [[0.376953125]]}}, "language_model.model.layers.14.self_attn.qkv_proj": {"inputs": [[[18.75]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.14.self_attn.o_proj": {"inputs": [[[5.09375]]], "outputs": [[[2.140625]], [[4.875]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.14.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.97265625]], [[16.0]]]}, "language_model.model.layers.14.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.1875]]]}, "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.97265625]]]}, "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.09375]]]}, "language_model.model.layers.14.self_attn.attn.impl.k_cache": {"inputs": [[[16.0]]]}, "language_model.model.layers.14.self_attn.attn.impl.v_cache": {"inputs": [[[5.1875]]]}, "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.25]], [[16.0]], [[5.1875]]], "outputs": [[[4.9375]], [[1.0]]]}, "language_model.model.layers.14.feed_forward.gate_up_proj": {"inputs": [[[12.0625]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.14.feed_forward.down_proj": {"inputs": [[[21.5]]], "outputs": [[[4.65625]], [[28.25]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.15.self_attn.qkv_proj": {"inputs": [[[20.875]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.15.self_attn.o_proj": {"inputs": [[[4.8125]]], "outputs": [[[21.875]], [[23.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.15.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.63671875]], [[10.3125]]]}, "language_model.model.layers.15.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.625]]]}, "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.63671875]]]}, "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.65625]]]}, "language_model.model.layers.15.self_attn.attn.impl.k_cache": {"inputs": [[[10.3125]]]}, "language_model.model.layers.15.self_attn.attn.impl.v_cache": {"inputs": [[[5.625]]]}, "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[7.90625]], [[10.3125]], [[5.625]]], "outputs": [[[4.8125]], [[1.0]]]}, "language_model.model.layers.15.feed_forward.router": {"inputs": [[[19.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op": {"inputs": [[[19.0]]], "outputs": [[[5.09375]], [[11.9375]], [[16.375]], [[36.5]], [[17.0]], [[22.125]], [[44.75]], [[25.75]], [[30.25]], [[56.5]], [[45.75]], [[27.5]], [[34.0]], [[26.875]], [[21.375]], [[6.9375]], [[21.625]], [[24.25]], [[37.75]], [[14.8125]], [[8.125]], [[35.75]], [[30.125]], [[32.25]], [[42.25]], [[18.5]], [[15.625]], [[24.0]], [[40.75]], [[35.0]], [[17.5]], [[35.0]], [[33.25]], [[23.375]], [[12.75]], [[29.375]], [[20.25]], [[7.46875]], [[30.875]], [[71.0]], [[42.75]], [[62.25]], [[51.5]], [[37.5]], [[72.0]], [[21.5]], [[29.75]], [[12.3125]], [[33.75]], [[14.5625]], [[21.5]], [[22.625]], [[51.5]], [[31.25]], [[31.125]], [[50.75]], [[69.5]], [[24.125]], [[34.5]], [[36.0]], [[74.0]], [[18.375]], [[55.25]], [[39.75]], [[78.5]], [[18.875]], [[40.5]], [[40.5]], [[56.5]], [[13.0625]], [[40.25]], [[51.75]], [[13.5625]], [[41.0]], [[56.5]], [[53.5]], [[45.0]], [[21.875]], [[31.875]], [[31.625]], [[14.625]], [[29.125]], [[23.625]], [[13.8125]], [[19.375]], [[17.0]], [[49.0]], [[35.5]], [[13.5]], [[39.75]], [[59.25]], [[9.9375]], [[25.625]], [[19.375]], [[22.5]], [[71.5]], [[29.875]], [[23.5]], [[16.25]], [[19.75]], [[27.5]], [[34.75]], [[38.25]], [[40.25]], [[32.75]], [[39.0]], [[28.625]], [[10.9375]], [[31.125]], [[21.5]], [[22.0]], [[34.5]], [[59.5]], [[61.0]], [[7.0625]], [[19.75]], [[12.9375]], [[70.0]], [[11.0]], [[25.875]], [[29.25]], [[76.0]], [[27.5]], [[42.5]], [[39.75]], [[51.5]], [[13.4375]], [[15.5]], [[34.5]]]}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[19.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.15.feed_forward.shared_expert.down_proj": {"inputs": [[[17.375]]], "outputs": [[[1.265625]], [[2.1533493531715637e+38]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.16.self_attn.qkv_proj": {"inputs": [[[20.375]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.16.self_attn.o_proj": {"inputs": [[[3.65625]]], "outputs": [[[2.75]], [[6.28125]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.16.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.69140625]], [[12.375]]]}, "language_model.model.layers.16.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.375]]]}, "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.69140625]]]}, "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.84375]]]}, "language_model.model.layers.16.self_attn.attn.impl.k_cache": {"inputs": [[[12.375]]]}, "language_model.model.layers.16.self_attn.attn.impl.v_cache": {"inputs": [[[4.375]]]}, "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.0625]], [[12.375]], [[4.375]]], "outputs": [[[3.65625]], [[1.0]]]}, "language_model.model.layers.16.feed_forward.gate_up_proj": {"inputs": [[[14.8125]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.16.feed_forward.down_proj": {"inputs": [[[18.625]]], "outputs": [[[7.03125]], [[22.625]]], "params": {"weight": [[0.57421875]]}}, "language_model.model.layers.17.self_attn.qkv_proj": {"inputs": [[[19.875]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.17.self_attn.o_proj": {"inputs": [[[3.015625]]], "outputs": [[[17.5]], [[20.75]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.17.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.82421875]], [[12.4375]]]}, "language_model.model.layers.17.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.5625]]]}, "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.82421875]]]}, "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.640625]]]}, "language_model.model.layers.17.self_attn.attn.impl.k_cache": {"inputs": [[[12.4375]]]}, "language_model.model.layers.17.self_attn.attn.impl.v_cache": {"inputs": [[[3.578125]]]}, "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.875]], [[12.4375]], [[3.578125]]], "outputs": [[[3.015625]], [[1.0]]]}, "language_model.model.layers.17.feed_forward.router": {"inputs": [[[23.875]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op": {"inputs": [[[23.875]]], "outputs": [[[5.84375]], [[29.625]], [[18.375]], [[47.5]], [[25.125]], [[53.25]], [[33.5]], [[103.0]], [[19.875]], [[47.75]], [[5.15625]], [[21.75]], [[26.375]], [[16.125]], [[25.875]], [[36.0]], [[30.875]], [[35.5]], [[30.625]], [[27.5]], [[36.25]], [[30.625]], [[38.25]], [[55.75]], [[51.25]], [[9.9375]], [[42.5]], [[13.8125]], [[32.25]], [[57.25]], [[39.25]], [[39.5]], [[26.125]], [[12.3125]], [[30.875]], [[16.75]], [[50.75]], [[41.25]], [[13.1875]], [[12.3125]], [[46.5]], [[16.5]], [[40.25]], [[5.4375]], [[32.0]], [[39.5]], [[60.5]], [[38.25]], [[72.0]], [[26.75]], [[32.25]], [[34.25]], [[46.5]], [[14.125]], [[14.0]], [[17.625]], [[11.75]], [[7.3125]], [[34.25]], [[30.375]], [[50.5]], [[14.0]], [[35.75]], [[25.75]], [[17.375]], [[50.0]], [[26.125]], [[18.25]], [[32.75]], [[21.625]], [[10.25]], [[22.0]], [[10.25]], [[50.75]], [[48.75]], [[79.5]], [[15.5]], [[62.25]], [[41.0]], [[11.0]], [[37.75]], [[47.0]], [[51.5]], [[42.25]], [[65.0]], [[47.0]], [[33.25]], [[28.0]], [[22.75]], [[46.25]], [[59.75]], [[64.5]], [[49.75]], [[8.8125]], [[29.0]], [[29.5]], [[49.25]], [[9.6875]], [[5.28125]], [[37.25]], [[24.5]], [[20.0]], [[24.375]], [[25.125]], [[40.75]], [[39.5]], [[27.125]], [[24.75]], [[57.75]], [[24.875]], [[19.625]], [[43.25]], [[34.5]], [[46.0]], [[31.5]], [[19.875]], [[18.25]], [[36.25]], [[25.25]], [[25.125]], [[46.5]], [[40.75]], [[29.125]], [[17.125]], [[17.25]], [[28.375]], [[4.46875]], [[11.5]], [[39.0]]]}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.365234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.875]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.17.feed_forward.shared_expert.down_proj": {"inputs": [[[21.625]]], "outputs": [[[10.8125]], [[2.1533493531715637e+38]]], "params": {"weight": [[0.53125]]}}, "language_model.model.layers.18.self_attn.qkv_proj": {"inputs": [[[22.125]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.18.self_attn.o_proj": {"inputs": [[[7.75]]], "outputs": [[[6.375]], [[11.5625]]], "params": {"weight": [[0.4765625]]}}, "language_model.model.layers.18.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0234375]], [[16.625]]]}, "language_model.model.layers.18.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.78125]]]}, "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0234375]]]}, "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.9375]]]}, "language_model.model.layers.18.self_attn.attn.impl.k_cache": {"inputs": [[[16.625]]]}, "language_model.model.layers.18.self_attn.attn.impl.v_cache": {"inputs": [[[7.78125]]]}, "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.1875]], [[16.625]], [[7.78125]]], "outputs": [[[7.75]], [[1.0]]]}, "language_model.model.layers.18.feed_forward.gate_up_proj": {"inputs": [[[18.5]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.18.feed_forward.down_proj": {"inputs": [[[18.25]]], "outputs": [[[8.25]], [[358.0]]], "params": {"weight": [[0.6875]]}}, "language_model.model.layers.19.self_attn.qkv_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.19.self_attn.o_proj": {"inputs": [[[4.09375]]], "outputs": [[[31.625]], [[36.75]]], "params": {"weight": [[0.6796875]]}}, "language_model.model.layers.19.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.76171875]], [[12.625]]]}, "language_model.model.layers.19.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.53125]]]}, "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.76171875]]]}, "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.65625]]]}, "language_model.model.layers.19.self_attn.attn.impl.k_cache": {"inputs": [[[12.625]]]}, "language_model.model.layers.19.self_attn.attn.impl.v_cache": {"inputs": [[[5.53125]]]}, "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.4375]], [[12.625]], [[5.53125]]], "outputs": [[[4.09375]], [[1.0]]]}, "language_model.model.layers.19.feed_forward.router": {"inputs": [[[23.25]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op": {"inputs": [[[23.25]]], "outputs": [[[5.09375]], [[43.25]], [[37.25]], [[12.1875]], [[37.0]], [[46.0]], [[34.75]], [[21.375]], [[42.75]], [[18.5]], [[22.5]], [[10.5]], [[81.5]], [[24.875]], [[15.25]], [[29.25]], [[7.53125]], [[9.0625]], [[27.125]], [[24.125]], [[43.5]], [[16.875]], [[25.875]], [[33.5]], [[20.25]], [[39.5]], [[37.0]], [[27.75]], [[45.25]], [[22.125]], [[25.125]], [[27.25]], [[63.25]], [[42.25]], [[17.25]], [[19.125]], [[44.5]], [[12.0]], [[29.5]], [[26.375]], [[17.375]], [[49.75]], [[11.125]], [[11.5]], [[45.0]], [[27.375]], [[19.125]], [[40.5]], [[21.0]], [[25.875]], [[45.5]], [[12.9375]], [[36.0]], [[5.03125]], [[46.75]], [[32.25]], [[14.5]], [[15.3125]], [[39.75]], [[40.5]], [[46.25]], [[17.5]], [[58.25]], [[11.25]], [[23.5]], [[31.0]], [[62.25]], [[23.0]], [[12.625]], [[65.0]], [[14.125]], [[58.0]], [[37.75]], [[32.25]], [[35.25]], [[23.25]], [[31.625]], [[21.125]], [[37.5]], [[38.75]], [[15.25]], [[73.0]], [[19.625]], [[17.125]], [[47.25]], [[38.0]], [[86.0]], [[12.5625]], [[9.6875]], [[31.125]], [[29.875]], [[22.25]], [[47.75]], [[24.125]], [[30.375]], [[25.25]], [[32.75]], [[24.25]], [[20.125]], [[26.625]], [[10.3125]], [[60.5]], [[26.25]], [[18.0]], [[32.75]], [[40.5]], [[23.375]], [[34.75]], [[24.5]], [[33.25]], [[28.125]], [[31.75]], [[13.125]], [[38.5]], [[22.875]], [[10.625]], [[17.625]], [[25.875]], [[48.25]], [[24.5]], [[50.75]], [[15.5625]], [[10.75]], [[24.125]], [[24.375]], [[15.5625]], [[15.9375]], [[24.5]], [[44.0]]]}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.3828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.44921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.89453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.25]]], "params": {"weight": [[0.35546875]]}}, "language_model.model.layers.19.feed_forward.shared_expert.down_proj": {"inputs": [[[27.5]]], "outputs": [[[6.625]], [[9.902748568597623e+37]]], "params": {"weight": [[0.455078125]]}}, "language_model.model.layers.20.self_attn.qkv_proj": {"inputs": [[[52.5]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.20.self_attn.o_proj": {"inputs": [[[4.59375]]], "outputs": [[[9.5]], [[18.375]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.20.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.921875]], [[18.125]]]}, "language_model.model.layers.20.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.75]]]}, "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.921875]]]}, "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.84375]]]}, "language_model.model.layers.20.self_attn.attn.impl.k_cache": {"inputs": [[[18.125]]]}, "language_model.model.layers.20.self_attn.attn.impl.v_cache": {"inputs": [[[7.75]]]}, "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.4375]], [[18.125]], [[7.75]]], "outputs": [[[4.59375]], [[1.0]]]}, "language_model.model.layers.20.feed_forward.gate_up_proj": {"inputs": [[[41.5]]], "params": {"weight": [[0.38671875]]}}, "language_model.model.layers.20.feed_forward.down_proj": {"inputs": [[[13.9375]]], "outputs": [[[9.4375]], [[38.5]]], "params": {"weight": [[0.77734375]]}}, "language_model.model.layers.21.self_attn.qkv_proj": {"inputs": [[[56.25]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.21.self_attn.o_proj": {"inputs": [[[4.59375]]], "outputs": [[[25.125]], [[37.75]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.21.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.94921875]], [[18.125]]]}, "language_model.model.layers.21.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.4375]]]}, "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.94921875]]]}, "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.0625]]]}, "language_model.model.layers.21.self_attn.attn.impl.k_cache": {"inputs": [[[18.125]]]}, "language_model.model.layers.21.self_attn.attn.impl.v_cache": {"inputs": [[[5.4375]]]}, "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.875]], [[18.125]], [[5.4375]]], "outputs": [[[4.59375]], [[1.0]]]}, "language_model.model.layers.21.feed_forward.router": {"inputs": [[[24.5]]], "params": {"weight": [[0.45703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op": {"inputs": [[[24.5]]], "outputs": [[[5.9375]], [[17.375]], [[58.5]], [[38.0]], [[63.75]], [[9.0625]], [[47.75]], [[23.625]], [[25.75]], [[49.0]], [[35.5]], [[17.125]], [[7.0]], [[18.75]], [[55.5]], [[21.0]], [[36.5]], [[41.5]], [[22.125]], [[25.125]], [[51.0]], [[37.25]], [[39.5]], [[35.5]], [[20.375]], [[44.5]], [[11.75]], [[10.125]], [[12.0625]], [[27.125]], [[44.5]], [[9.5]], [[57.0]], [[55.75]], [[10.875]], [[23.375]], [[10.5625]], [[25.625]], [[16.25]], [[38.0]], [[44.0]], [[5.96875]], [[48.25]], [[24.125]], [[13.8125]], [[34.0]], [[23.125]], [[8.3125]], [[40.0]], [[21.0]], [[32.25]], [[56.0]], [[47.25]], [[32.0]], [[39.5]], [[13.25]], [[30.375]], [[28.75]], [[19.25]], [[9.25]], [[39.0]], [[70.0]], [[11.625]], [[23.125]], [[49.0]], [[11.25]], [[63.25]], [[13.6875]], [[22.0]], [[8.3125]], [[16.0]], [[35.0]], [[15.625]], [[42.75]], [[43.5]], [[18.25]], [[12.125]], [[47.0]], [[11.8125]], [[24.875]], [[15.5]], [[25.5]], [[31.25]], [[79.5]], [[42.25]], [[25.25]], [[29.875]], [[8.8125]], [[18.125]], [[7.21875]], [[18.125]], [[20.875]], [[8.75]], [[38.0]], [[12.125]], [[11.25]], [[27.375]], [[20.75]], [[17.875]], [[23.5]], [[21.875]], [[29.75]], [[75.5]], [[27.875]], [[56.25]], [[28.5]], [[25.0]], [[25.125]], [[37.75]], [[40.25]], [[19.0]], [[34.5]], [[41.75]], [[40.5]], [[11.1875]], [[36.5]], [[17.25]], [[21.25]], [[13.25]], [[28.0]], [[35.5]], [[8.1875]], [[26.125]], [[25.125]], [[18.625]], [[12.125]], [[47.25]], [[32.25]], [[71.5]]]}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[24.5]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.21.feed_forward.shared_expert.down_proj": {"inputs": [[[21.5]]], "outputs": [[[16.375]], [[2.3053798051894635e+36]]], "params": {"weight": [[0.890625]]}}, "language_model.model.layers.22.self_attn.qkv_proj": {"inputs": [[[55.75]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.22.self_attn.o_proj": {"inputs": [[[4.3125]]], "outputs": [[[17.25]], [[23.25]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.22.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.03125]], [[17.625]]]}, "language_model.model.layers.22.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.21875]]]}, "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.03125]]]}, "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.9375]]]}, "language_model.model.layers.22.self_attn.attn.impl.k_cache": {"inputs": [[[17.625]]]}, "language_model.model.layers.22.self_attn.attn.impl.v_cache": {"inputs": [[[6.21875]]]}, "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.75]], [[17.625]], [[6.21875]]], "outputs": [[[4.3125]], [[1.0]]]}, "language_model.model.layers.22.feed_forward.gate_up_proj": {"inputs": [[[44.75]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.22.feed_forward.down_proj": {"inputs": [[[17.875]]], "outputs": [[[12.0]], [[80.5]]], "params": {"weight": [[0.734375]]}}, "language_model.model.layers.23.self_attn.qkv_proj": {"inputs": [[[67.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.23.self_attn.o_proj": {"inputs": [[[7.40625]]], "outputs": [[[50.5]], [[61.5]]], "params": {"weight": [[0.388671875]]}}, "language_model.model.layers.23.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.125]], [[15.3125]]]}, "language_model.model.layers.23.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.59375]]]}, "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.125]]]}, "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.5]]]}, "language_model.model.layers.23.self_attn.attn.impl.k_cache": {"inputs": [[[15.3125]]]}, "language_model.model.layers.23.self_attn.attn.impl.v_cache": {"inputs": [[[7.59375]]]}, "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.3125]], [[15.3125]], [[7.59375]]], "outputs": [[[7.40625]], [[1.0]]]}, "language_model.model.layers.23.feed_forward.router": {"inputs": [[[23.625]]], "params": {"weight": [[0.474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op": {"inputs": [[[23.625]]], "outputs": [[[18.75]], [[56.5]], [[16.5]], [[49.75]], [[44.75]], [[7.71875]], [[53.75]], [[21.375]], [[29.75]], [[71.5]], [[26.125]], [[28.75]], [[62.0]], [[36.5]], [[14.125]], [[46.25]], [[73.0]], [[24.5]], [[20.375]], [[27.375]], [[48.5]], [[17.25]], [[42.25]], [[8.4375]], [[30.0]], [[53.5]], [[20.625]], [[35.0]], [[115.5]], [[26.5]], [[19.25]], [[21.625]], [[16.375]], [[19.375]], [[12.0625]], [[8.6875]], [[33.25]], [[15.25]], [[13.625]], [[26.875]], [[41.5]], [[79.0]], [[8.6875]], [[32.5]], [[9.25]], [[16.25]], [[10.625]], [[51.25]], [[63.75]], [[8.125]], [[70.0]], [[34.75]], [[46.25]], [[40.5]], [[20.0]], [[20.5]], [[20.5]], [[93.5]], [[8.0625]], [[68.0]], [[27.25]], [[15.125]], [[40.0]], [[11.5]], [[10.8125]], [[17.375]], [[8.5]], [[74.0]], [[44.5]], [[52.0]], [[58.75]], [[6.84375]], [[35.5]], [[53.25]], [[44.0]], [[11.4375]], [[33.75]], [[38.25]], [[86.5]], [[22.0]], [[26.5]], [[20.5]], [[18.875]], [[54.5]], [[34.75]], [[6.0]], [[46.0]], [[11.8125]], [[7.25]], [[24.625]], [[14.125]], [[38.0]], [[43.0]], [[26.5]], [[27.375]], [[35.5]], [[45.25]], [[25.0]], [[29.25]], [[51.5]], [[45.75]], [[13.4375]], [[25.125]], [[9.125]], [[6.59375]], [[29.625]], [[70.0]], [[54.5]], [[40.0]], [[9.6875]], [[97.0]], [[18.375]], [[81.0]], [[9.5]], [[33.5]], [[31.125]], [[40.0]], [[22.0]], [[45.75]], [[21.25]], [[14.5625]], [[14.625]], [[35.5]], [[59.75]], [[51.5]], [[21.375]], [[26.0]], [[9.9375]], [[10.3125]]]}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.625]]], "params": {"weight": [[0.703125]]}}, "language_model.model.layers.23.feed_forward.shared_expert.down_proj": {"inputs": [[[21.25]]], "outputs": [[[5.1875]], [[4.2369142365644193e+37]]], "params": {"weight": [[0.470703125]]}}, "language_model.model.layers.24.self_attn.qkv_proj": {"inputs": [[[60.5]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.24.self_attn.o_proj": {"inputs": [[[6.3125]]], "outputs": [[[36.0]], [[42.5]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.24.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.28125]], [[15.25]]]}, "language_model.model.layers.24.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.90625]]]}, "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.28125]]]}, "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.4375]]]}, "language_model.model.layers.24.self_attn.attn.impl.k_cache": {"inputs": [[[15.25]]]}, "language_model.model.layers.24.self_attn.attn.impl.v_cache": {"inputs": [[[7.90625]]]}, "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.5]], [[15.25]], [[7.90625]]], "outputs": [[[6.3125]], [[1.0]]]}, "language_model.model.layers.24.feed_forward.gate_up_proj": {"inputs": [[[40.5]]], "params": {"weight": [[0.380859375]]}}, "language_model.model.layers.24.feed_forward.down_proj": {"inputs": [[[22.125]]], "outputs": [[[13.3125]], [[62.5]]], "params": {"weight": [[0.72265625]]}}, "language_model.model.layers.25.self_attn.qkv_proj": {"inputs": [[[49.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.25.self_attn.o_proj": {"inputs": [[[5.28125]]], "outputs": [[[47.75]], [[57.0]]], "params": {"weight": [[0.47265625]]}}, "language_model.model.layers.25.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.2578125]], [[14.875]]]}, "language_model.model.layers.25.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.4375]]]}, "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.2578125]]]}, "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.40625]]]}, "language_model.model.layers.25.self_attn.attn.impl.k_cache": {"inputs": [[[14.875]]]}, "language_model.model.layers.25.self_attn.attn.impl.v_cache": {"inputs": [[[6.4375]]]}, "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[16.0]], [[14.875]], [[6.4375]]], "outputs": [[[5.28125]], [[1.0]]]}, "language_model.model.layers.25.feed_forward.router": {"inputs": [[[30.375]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op": {"inputs": [[[30.375]]], "outputs": [[[6.5625]], [[53.25]], [[36.5]], [[26.5]], [[33.5]], [[26.0]], [[38.25]], [[12.625]], [[48.25]], [[14.4375]], [[28.75]], [[7.375]], [[30.0]], [[29.75]], [[49.25]], [[73.0]], [[31.625]], [[12.0625]], [[17.125]], [[55.5]], [[35.75]], [[46.75]], [[49.5]], [[47.0]], [[51.75]], [[8.3125]], [[26.75]], [[49.0]], [[24.875]], [[9.6875]], [[40.5]], [[27.875]], [[17.625]], [[14.125]], [[57.75]], [[16.625]], [[13.3125]], [[27.75]], [[21.625]], [[12.0625]], [[59.5]], [[22.0]], [[52.0]], [[35.25]], [[50.25]], [[29.875]], [[57.25]], [[99.5]], [[14.25]], [[17.5]], [[53.75]], [[36.75]], [[9.5]], [[15.9375]], [[10.5625]], [[31.125]], [[18.375]], [[25.75]], [[19.875]], [[39.25]], [[9.5]], [[27.375]], [[37.25]], [[21.375]], [[15.5]], [[79.0]], [[14.5]], [[32.75]], [[48.75]], [[47.0]], [[14.75]], [[15.125]], [[15.625]], [[32.75]], [[23.625]], [[72.0]], [[67.5]], [[15.4375]], [[8.125]], [[38.5]], [[32.75]], [[19.625]], [[34.75]], [[30.375]], [[52.5]], [[10.5]], [[26.875]], [[41.25]], [[46.0]], [[29.25]], [[52.0]], [[13.375]], [[15.75]], [[32.25]], [[35.5]], [[9.9375]], [[50.25]], [[25.75]], [[29.625]], [[11.9375]], [[25.375]], [[40.5]], [[42.25]], [[33.25]], [[20.375]], [[14.375]], [[19.75]], [[37.5]], [[5.34375]], [[36.25]], [[25.625]], [[38.5]], [[23.375]], [[38.25]], [[38.25]], [[49.0]], [[50.5]], [[19.875]], [[24.25]], [[70.0]], [[37.25]], [[28.125]], [[35.75]], [[34.75]], [[27.875]], [[61.0]], [[13.0625]], [[24.25]], [[10.5625]]]}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[30.375]]], "params": {"weight": [[0.419921875]]}}, "language_model.model.layers.25.feed_forward.shared_expert.down_proj": {"inputs": [[[28.625]]], "outputs": [[[24.125]], [[3.1502703500102506e+38]]], "params": {"weight": [[0.80859375]]}}, "language_model.model.layers.26.self_attn.qkv_proj": {"inputs": [[[63.25]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.26.self_attn.o_proj": {"inputs": [[[6.96875]]], "outputs": [[[29.0]], [[53.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.26.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1015625]], [[19.25]]]}, "language_model.model.layers.26.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.0625]]]}, "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1015625]]]}, "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[8.0625]]]}, "language_model.model.layers.26.self_attn.attn.impl.k_cache": {"inputs": [[[19.25]]]}, "language_model.model.layers.26.self_attn.attn.impl.v_cache": {"inputs": [[[9.0625]]]}, "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.75]], [[19.25]], [[9.0625]]], "outputs": [[[6.96875]], [[1.0]]]}, "language_model.model.layers.26.feed_forward.gate_up_proj": {"inputs": [[[45.5]]], "params": {"weight": [[0.359375]]}}, "language_model.model.layers.26.feed_forward.down_proj": {"inputs": [[[24.625]]], "outputs": [[[15.25]], [[103.0]]], "params": {"weight": [[0.66796875]]}}, "language_model.model.layers.27.self_attn.qkv_proj": {"inputs": [[[72.5]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.27.self_attn.o_proj": {"inputs": [[[5.4375]]], "outputs": [[[60.0]], [[66.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.27.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.87109375]], [[14.875]]]}, "language_model.model.layers.27.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[13.3125]]]}, "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.87109375]]]}, "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.0]]]}, "language_model.model.layers.27.self_attn.attn.impl.k_cache": {"inputs": [[[14.875]]]}, "language_model.model.layers.27.self_attn.attn.impl.v_cache": {"inputs": [[[13.3125]]]}, "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.875]], [[14.875]], [[13.3125]]], "outputs": [[[5.4375]], [[1.0]]]}, "language_model.model.layers.27.feed_forward.router": {"inputs": [[[30.875]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op": {"inputs": [[[30.875]]], "outputs": [[[6.84375]], [[34.75]], [[37.5]], [[49.5]], [[8.3125]], [[9.9375]], [[28.75]], [[50.0]], [[30.75]], [[39.75]], [[27.0]], [[33.5]], [[29.25]], [[33.0]], [[17.125]], [[66.0]], [[22.875]], [[16.75]], [[99.5]], [[27.125]], [[17.875]], [[18.5]], [[11.3125]], [[44.0]], [[25.125]], [[25.875]], [[20.625]], [[35.25]], [[25.125]], [[28.625]], [[30.875]], [[78.5]], [[13.0]], [[81.5]], [[30.0]], [[78.0]], [[18.25]], [[36.5]], [[61.0]], [[81.5]], [[24.625]], [[21.125]], [[49.25]], [[70.5]], [[36.5]], [[13.375]], [[24.875]], [[23.125]], [[21.5]], [[83.0]], [[11.5625]], [[29.875]], [[8.0625]], [[32.0]], [[42.75]], [[35.25]], [[41.5]], [[37.0]], [[37.5]], [[45.25]], [[10.3125]], [[27.75]], [[35.75]], [[45.0]], [[13.4375]], [[41.0]], [[78.5]], [[11.5625]], [[27.625]], [[14.8125]], [[13.9375]], [[20.75]], [[37.0]], [[77.0]], [[38.25]], [[36.25]], [[72.5]], [[35.5]], [[31.25]], [[48.75]], [[35.5]], [[69.5]], [[44.25]], [[28.375]], [[41.5]], [[24.125]], [[21.625]], [[15.8125]], [[17.375]], [[17.0]], [[24.625]], [[17.375]], [[48.5]], [[14.125]], [[44.75]], [[35.25]], [[28.375]], [[46.25]], [[27.5]], [[13.75]], [[23.5]], [[54.25]], [[31.625]], [[16.0]], [[24.75]], [[11.3125]], [[11.625]], [[12.5]], [[44.0]], [[11.5625]], [[57.0]], [[16.75]], [[46.75]], [[13.625]], [[17.625]], [[25.125]], [[21.25]], [[29.875]], [[25.75]], [[35.0]], [[123.0]], [[14.5]], [[52.5]], [[33.0]], [[86.5]], [[16.25]], [[31.0]], [[14.75]], [[26.0]]]}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.3046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[30.875]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.27.feed_forward.shared_expert.down_proj": {"inputs": [[[39.75]]], "outputs": [[[12.75]], [[9.902748568597623e+37]]], "params": {"weight": [[0.83984375]]}}, "language_model.model.layers.28.self_attn.qkv_proj": {"inputs": [[[77.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.28.self_attn.o_proj": {"inputs": [[[5.3125]]], "outputs": [[[72.0]], [[74.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.28.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.375]], [[15.625]]]}, "language_model.model.layers.28.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.0625]]]}, "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.375]]]}, "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.03125]]]}, "language_model.model.layers.28.self_attn.attn.impl.k_cache": {"inputs": [[[15.625]]]}, "language_model.model.layers.28.self_attn.attn.impl.v_cache": {"inputs": [[[7.0625]]]}, "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[18.625]], [[15.625]], [[7.0625]]], "outputs": [[[5.3125]], [[1.0]]]}, "language_model.model.layers.28.feed_forward.gate_up_proj": {"inputs": [[[40.25]]], "params": {"weight": [[0.431640625]]}}, "language_model.model.layers.28.feed_forward.down_proj": {"inputs": [[[38.25]]], "outputs": [[[95.0]], [[103.5]]], "params": {"weight": [[0.36328125]]}}, "language_model.model.layers.29.self_attn.qkv_proj": {"inputs": [[[66.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.29.self_attn.o_proj": {"inputs": [[[8.6875]]], "outputs": [[[70.5]], [[76.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.29.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.96484375]], [[15.9375]]]}, "language_model.model.layers.29.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.9375]]]}, "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.96484375]]]}, "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.8125]]]}, "language_model.model.layers.29.self_attn.attn.impl.k_cache": {"inputs": [[[15.9375]]]}, "language_model.model.layers.29.self_attn.attn.impl.v_cache": {"inputs": [[[11.9375]]]}, "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.5625]], [[15.9375]], [[11.9375]]], "outputs": [[[8.6875]], [[1.0]]]}, "language_model.model.layers.29.feed_forward.router": {"inputs": [[[31.75]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op": {"inputs": [[[31.75]]], "outputs": [[[6.46875]], [[9.875]], [[37.25]], [[38.75]], [[38.75]], [[40.75]], [[43.25]], [[14.3125]], [[41.5]], [[50.0]], [[28.125]], [[19.75]], [[13.8125]], [[17.75]], [[25.625]], [[33.5]], [[8.5]], [[57.5]], [[60.75]], [[31.25]], [[14.875]], [[22.25]], [[14.0]], [[21.75]], [[19.125]], [[61.0]], [[26.0]], [[34.25]], [[18.125]], [[16.75]], [[59.0]], [[33.75]], [[16.75]], [[43.0]], [[50.0]], [[21.625]], [[36.5]], [[25.25]], [[29.875]], [[69.5]], [[57.5]], [[15.9375]], [[20.125]], [[11.625]], [[63.5]], [[36.0]], [[24.25]], [[17.125]], [[32.75]], [[64.5]], [[21.5]], [[17.0]], [[26.375]], [[53.75]], [[67.0]], [[42.0]], [[44.25]], [[34.25]], [[31.0]], [[34.25]], [[15.0625]], [[49.5]], [[11.0]], [[34.25]], [[26.375]], [[64.5]], [[48.0]], [[25.875]], [[54.75]], [[41.5]], [[50.25]], [[24.0]], [[34.25]], [[20.5]], [[40.25]], [[40.0]], [[64.0]], [[19.25]], [[51.75]], [[34.5]], [[16.25]], [[34.0]], [[31.25]], [[22.25]], [[40.75]], [[22.125]], [[40.0]], [[21.25]], [[11.6875]], [[52.75]], [[24.75]], [[29.25]], [[23.125]], [[31.0]], [[22.875]], [[7.46875]], [[31.625]], [[22.0]], [[18.5]], [[46.0]], [[21.0]], [[24.75]], [[33.0]], [[16.5]], [[50.5]], [[8.875]], [[88.0]], [[53.5]], [[14.5]], [[17.875]], [[38.0]], [[32.0]], [[45.75]], [[49.25]], [[15.75]], [[23.125]], [[40.0]], [[35.25]], [[45.0]], [[77.0]], [[23.875]], [[38.0]], [[27.875]], [[45.0]], [[97.0]], [[14.5]], [[51.0]], [[103.5]], [[20.0]]]}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[31.75]]], "params": {"weight": [[0.41015625]]}}, "language_model.model.layers.29.feed_forward.shared_expert.down_proj": {"inputs": [[[39.25]]], "outputs": [[[11.0]], [[2.1044980667851465e+28]]], "params": {"weight": [[0.87109375]]}}, "language_model.model.layers.30.self_attn.qkv_proj": {"inputs": [[[71.5]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.30.self_attn.o_proj": {"inputs": [[[6.65625]]], "outputs": [[[52.25]], [[57.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.30.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.99609375]], [[18.875]]]}, "language_model.model.layers.30.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.625]]]}, "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.99609375]]]}, "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.21875]]]}, "language_model.model.layers.30.self_attn.attn.impl.k_cache": {"inputs": [[[18.875]]]}, "language_model.model.layers.30.self_attn.attn.impl.v_cache": {"inputs": [[[8.625]]]}, "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.0]], [[18.875]], [[8.625]]], "outputs": [[[6.65625]], [[1.0]]]}, "language_model.model.layers.30.feed_forward.gate_up_proj": {"inputs": [[[32.0]]], "params": {"weight": [[0.392578125]]}}, "language_model.model.layers.30.feed_forward.down_proj": {"inputs": [[[112.0]]], "outputs": [[[18.625]], [[139.0]]], "params": {"weight": [[0.640625]]}}, "language_model.model.layers.31.self_attn.qkv_proj": {"inputs": [[[71.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.31.self_attn.o_proj": {"inputs": [[[7.25]]], "outputs": [[[66.5]], [[88.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.31.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.98828125]], [[15.0]]]}, "language_model.model.layers.31.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.5625]]]}, "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.98828125]]]}, "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.1875]]]}, "language_model.model.layers.31.self_attn.attn.impl.k_cache": {"inputs": [[[15.0]]]}, "language_model.model.layers.31.self_attn.attn.impl.v_cache": {"inputs": [[[12.5625]]]}, "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.3125]], [[15.0]], [[12.5625]]], "outputs": [[[6.78125]], [[1.0]]]}, "language_model.model.layers.31.feed_forward.router": {"inputs": [[[33.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op": {"inputs": [[[33.0]]], "outputs": [[[6.6875]], [[15.8125]], [[80.5]], [[20.625]], [[38.5]], [[40.25]], [[41.25]], [[54.25]], [[14.625]], [[23.875]], [[44.5]], [[44.25]], [[16.25]], [[20.0]], [[13.5]], [[23.875]], [[10.25]], [[23.625]], [[25.375]], [[85.5]], [[21.375]], [[10.0]], [[22.875]], [[28.0]], [[60.75]], [[76.5]], [[47.25]], [[37.25]], [[52.5]], [[18.25]], [[14.9375]], [[30.0]], [[35.5]], [[35.0]], [[61.25]], [[12.375]], [[15.125]], [[29.875]], [[22.0]], [[26.75]], [[90.0]], [[53.5]], [[45.5]], [[26.125]], [[45.0]], [[48.5]], [[71.0]], [[52.25]], [[44.25]], [[23.25]], [[32.5]], [[57.75]], [[71.0]], [[66.0]], [[12.0625]], [[36.25]], [[32.75]], [[17.625]], [[92.0]], [[29.75]], [[21.375]], [[27.0]], [[57.5]], [[20.5]], [[45.75]], [[43.75]], [[25.75]], [[23.625]], [[31.0]], [[45.0]], [[17.125]], [[49.75]], [[38.0]], [[50.0]], [[44.0]], [[14.375]], [[12.5]], [[51.5]], [[49.25]], [[39.5]], [[48.5]], [[50.25]], [[65.5]], [[58.0]], [[92.0]], [[18.25]], [[31.625]], [[86.5]], [[24.625]], [[54.0]], [[56.25]], [[51.25]], [[37.0]], [[71.5]], [[17.25]], [[48.25]], [[40.75]], [[42.0]], [[63.25]], [[17.375]], [[25.75]], [[21.625]], [[18.625]], [[65.5]], [[30.25]], [[15.9375]], [[7.0625]], [[57.5]], [[11.625]], [[27.25]], [[60.75]], [[62.25]], [[60.5]], [[49.25]], [[30.125]], [[37.0]], [[37.0]], [[66.5]], [[17.375]], [[27.0]], [[44.25]], [[43.25]], [[24.375]], [[12.5625]], [[14.5]], [[50.75]], [[14.6875]], [[22.625]], [[35.0]]]}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.42578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[33.0]]], "params": {"weight": [[0.455078125]]}}, "language_model.model.layers.31.feed_forward.shared_expert.down_proj": {"inputs": [[[47.75]]], "outputs": [[[7.9375]], [[9.902748568597623e+37]]], "params": {"weight": [[0.55859375]]}}, "language_model.model.layers.32.self_attn.qkv_proj": {"inputs": [[[53.25]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.32.self_attn.o_proj": {"inputs": [[[5.96875]]], "outputs": [[[96.0]], [[105.0]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.32.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.484375]], [[17.5]]]}, "language_model.model.layers.32.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.71875]]]}, "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.484375]]]}, "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.84375]]]}, "language_model.model.layers.32.self_attn.attn.impl.k_cache": {"inputs": [[[17.5]]]}, "language_model.model.layers.32.self_attn.attn.impl.v_cache": {"inputs": [[[7.84375]]]}, "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[18.625]], [[17.5]], [[7.84375]]], "outputs": [[[5.8125]], [[1.0]]]}, "language_model.model.layers.32.feed_forward.gate_up_proj": {"inputs": [[[32.75]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.32.feed_forward.down_proj": {"inputs": [[[43.0]]], "outputs": [[[21.125]], [[79.5]]], "params": {"weight": [[0.5625]]}}, "language_model.model.layers.33.self_attn.qkv_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.33.self_attn.o_proj": {"inputs": [[[8.125]]], "outputs": [[[97.0]], [[118.0]]], "params": {"weight": [[0.796875]]}}, "language_model.model.layers.33.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.5234375]], [[17.875]]]}, "language_model.model.layers.33.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.375]]]}, "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.5234375]]]}, "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[8.0]]]}, "language_model.model.layers.33.self_attn.attn.impl.k_cache": {"inputs": [[[17.875]]]}, "language_model.model.layers.33.self_attn.attn.impl.v_cache": {"inputs": [[[8.375]]]}, "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[18.0]], [[17.875]], [[8.125]]], "outputs": [[[8.125]], [[1.0]]]}, "language_model.model.layers.33.feed_forward.router": {"inputs": [[[64.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op": {"inputs": [[[64.0]]], "outputs": [[[12.4375]], [[65.0]], [[52.0]], [[37.0]], [[29.625]], [[30.125]], [[66.0]], [[15.1875]], [[118.0]], [[33.5]], [[126.0]], [[24.375]], [[47.0]], [[55.0]], [[25.125]], [[54.5]], [[54.25]], [[39.0]], [[33.25]], [[18.125]], [[20.375]], [[27.875]], [[20.0]], [[18.75]], [[33.5]], [[21.75]], [[53.0]], [[105.5]], [[29.375]], [[51.0]], [[40.5]], [[21.125]], [[50.0]], [[19.375]], [[58.25]], [[54.0]], [[18.375]], [[30.25]], [[21.375]], [[85.0]], [[31.875]], [[13.875]], [[24.0]], [[69.5]], [[102.5]], [[36.0]], [[21.75]], [[59.5]], [[18.375]], [[69.5]], [[58.25]], [[28.875]], [[13.875]], [[35.25]], [[28.375]], [[135.0]], [[99.0]], [[67.5]], [[20.625]], [[34.5]], [[91.5]], [[51.25]], [[26.25]], [[41.75]], [[84.5]], [[24.5]], [[41.25]], [[34.25]], [[19.5]], [[62.5]], [[58.75]], [[71.5]], [[61.25]], [[88.0]], [[94.0]], [[23.25]], [[67.0]], [[28.125]], [[30.625]], [[14.5]], [[29.25]], [[18.5]], [[12.5625]], [[60.5]], [[24.625]], [[16.25]], [[84.5]], [[39.25]], [[24.75]], [[74.5]], [[48.25]], [[20.5]], [[22.375]], [[22.375]], [[36.5]], [[97.0]], [[75.5]], [[28.0]], [[66.0]], [[35.25]], [[81.0]], [[76.5]], [[74.5]], [[50.0]], [[32.5]], [[81.5]], [[19.75]], [[56.75]], [[54.0]], [[21.875]], [[17.25]], [[55.0]], [[117.0]], [[12.8125]], [[41.75]], [[90.0]], [[33.0]], [[17.625]], [[50.25]], [[33.25]], [[89.5]], [[49.0]], [[39.0]], [[22.5]], [[8.5625]], [[57.0]], [[30.375]], [[14.375]], [[17.25]]]}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.431640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.357421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[64.0]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.33.feed_forward.shared_expert.down_proj": {"inputs": [[[63.25]]], "outputs": [[[12.3125]], [[4.2369142365644193e+37]]], "params": {"weight": [[0.470703125]]}}, "language_model.model.layers.34.self_attn.qkv_proj": {"inputs": [[[54.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.34.self_attn.o_proj": {"inputs": [[[8.0]]], "outputs": [[[46.75]], [[80.5]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.34.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.09375]], [[16.875]]]}, "language_model.model.layers.34.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[10.4375]]]}, "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.09375]]]}, "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.875]]]}, "language_model.model.layers.34.self_attn.attn.impl.k_cache": {"inputs": [[[16.875]]]}, "language_model.model.layers.34.self_attn.attn.impl.v_cache": {"inputs": [[[10.4375]]]}, "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.875]], [[16.875]], [[10.4375]]], "outputs": [[[8.0]], [[1.0]]]}, "language_model.model.layers.34.feed_forward.gate_up_proj": {"inputs": [[[31.125]]], "params": {"weight": [[0.490234375]]}}, "language_model.model.layers.34.feed_forward.down_proj": {"inputs": [[[74.5]]], "outputs": [[[30.375]], [[128.0]]], "params": {"weight": [[0.80859375]]}}, "language_model.model.layers.35.self_attn.qkv_proj": {"inputs": [[[52.75]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.35.self_attn.o_proj": {"inputs": [[[6.6875]]], "outputs": [[[87.5]], [[93.0]]], "params": {"weight": [[0.87109375]]}}, "language_model.model.layers.35.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0625]], [[13.3125]]]}, "language_model.model.layers.35.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.3125]]]}, "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0625]]]}, "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.1875]]]}, "language_model.model.layers.35.self_attn.attn.impl.k_cache": {"inputs": [[[13.3125]]]}, "language_model.model.layers.35.self_attn.attn.impl.v_cache": {"inputs": [[[12.3125]]]}, "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.75]], [[13.3125]], [[12.3125]]], "outputs": [[[6.6875]], [[1.0]]]}, "language_model.model.layers.35.feed_forward.router": {"inputs": [[[64.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op": {"inputs": [[[64.0]]], "outputs": [[[10.75]], [[125.5]], [[23.0]], [[23.125]], [[51.5]], [[155.0]], [[44.25]], [[80.0]], [[84.0]], [[66.5]], [[66.5]], [[89.0]], [[36.0]], [[101.5]], [[94.0]], [[84.5]], [[63.5]], [[61.0]], [[93.0]], [[100.0]], [[45.0]], [[52.0]], [[34.75]], [[24.875]], [[36.75]], [[33.25]], [[44.5]], [[84.5]], [[60.75]], [[37.5]], [[72.5]], [[121.5]], [[75.5]], [[59.25]], [[35.0]], [[48.5]], [[144.0]], [[42.75]], [[32.5]], [[37.75]], [[57.0]], [[64.5]], [[52.75]], [[31.25]], [[82.5]], [[84.0]], [[25.75]], [[23.875]], [[43.5]], [[62.75]], [[77.5]], [[31.125]], [[48.0]], [[59.75]], [[47.75]], [[20.875]], [[32.75]], [[86.0]], [[29.25]], [[50.5]], [[71.0]], [[69.5]], [[27.375]], [[57.75]], [[37.25]], [[43.75]], [[18.625]], [[83.5]], [[21.125]], [[27.875]], [[98.0]], [[54.25]], [[122.5]], [[30.875]], [[36.75]], [[40.0]], [[35.0]], [[127.0]], [[67.0]], [[129.0]], [[108.5]], [[127.5]], [[107.5]], [[57.75]], [[38.0]], [[78.5]], [[120.0]], [[36.5]], [[29.0]], [[24.0]], [[42.0]], [[75.5]], [[73.5]], [[79.5]], [[95.5]], [[115.0]], [[70.5]], [[85.0]], [[48.5]], [[117.0]], [[37.25]], [[99.5]], [[14.3125]], [[32.75]], [[46.0]], [[19.125]], [[66.5]], [[30.75]], [[39.25]], [[107.0]], [[60.25]], [[55.5]], [[39.25]], [[80.5]], [[28.125]], [[61.75]], [[79.5]], [[59.5]], [[21.125]], [[40.25]], [[39.75]], [[66.0]], [[22.5]], [[33.5]], [[88.0]], [[76.0]], [[36.5]], [[33.5]], [[44.75]]]}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.361328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.4140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.32421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[64.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.35.feed_forward.shared_expert.down_proj": {"inputs": [[[71.0]]], "outputs": [[[45.5]], [[3.1502703500102506e+38]]], "params": {"weight": [[0.875]]}}, "language_model.model.layers.36.self_attn.qkv_proj": {"inputs": [[[56.75]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.36.self_attn.o_proj": {"inputs": [[[10.875]]], "outputs": [[[27.75]], [[43.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.36.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.6875]], [[16.625]]]}, "language_model.model.layers.36.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.6875]]]}, "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.6875]]]}, "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[9.3125]]]}, "language_model.model.layers.36.self_attn.attn.impl.k_cache": {"inputs": [[[16.625]]]}, "language_model.model.layers.36.self_attn.attn.impl.v_cache": {"inputs": [[[12.6875]]]}, "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[22.125]], [[16.625]], [[12.6875]]], "outputs": [[[10.875]], [[1.0]]]}, "language_model.model.layers.36.feed_forward.gate_up_proj": {"inputs": [[[29.625]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.36.feed_forward.down_proj": {"inputs": [[[84.0]]], "outputs": [[[38.25]], [[76.5]]], "params": {"weight": [[0.6953125]]}}, "language_model.model.layers.37.self_attn.qkv_proj": {"inputs": [[[50.25]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.37.self_attn.o_proj": {"inputs": [[[8.5625]]], "outputs": [[[102.0]], [[105.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.37.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.2578125]], [[16.0]]]}, "language_model.model.layers.37.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[10.3125]]]}, "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.2578125]]]}, "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[9.0]]]}, "language_model.model.layers.37.self_attn.attn.impl.k_cache": {"inputs": [[[16.0]]]}, "language_model.model.layers.37.self_attn.attn.impl.v_cache": {"inputs": [[[10.3125]]]}, "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[16.375]], [[16.0]], [[10.3125]]], "outputs": [[[8.3125]], [[1.0]]]}, "language_model.model.layers.37.feed_forward.router": {"inputs": [[[70.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op": {"inputs": [[[70.0]]], "outputs": [[[12.4375]], [[98.5]], [[80.5]], [[32.5]], [[113.0]], [[64.0]], [[45.5]], [[55.5]], [[76.0]], [[30.375]], [[94.0]], [[36.0]], [[39.5]], [[101.0]], [[34.0]], [[53.0]], [[54.5]], [[82.5]], [[40.0]], [[59.5]], [[50.75]], [[49.0]], [[61.75]], [[77.0]], [[46.25]], [[109.0]], [[62.5]], [[77.5]], [[105.0]], [[113.5]], [[50.0]], [[34.5]], [[70.0]], [[34.0]], [[89.0]], [[86.0]], [[62.0]], [[71.0]], [[147.0]], [[72.5]], [[75.5]], [[68.5]], [[26.0]], [[35.0]], [[32.5]], [[52.0]], [[194.0]], [[132.0]], [[30.75]], [[83.5]], [[24.875]], [[50.25]], [[65.5]], [[87.5]], [[49.0]], [[29.375]], [[96.5]], [[147.0]], [[77.0]], [[57.0]], [[96.0]], [[85.5]], [[46.25]], [[60.25]], [[29.125]], [[86.5]], [[25.625]], [[49.0]], [[92.5]], [[36.0]], [[35.0]], [[41.5]], [[89.5]], [[41.75]], [[23.5]], [[22.5]], [[99.0]], [[148.0]], [[23.125]], [[71.5]], [[87.5]], [[74.0]], [[34.0]], [[115.0]], [[41.5]], [[49.25]], [[31.875]], [[75.0]], [[82.0]], [[137.0]], [[126.5]], [[24.125]], [[74.0]], [[34.5]], [[52.0]], [[30.25]], [[68.0]], [[22.5]], [[32.25]], [[78.0]], [[93.0]], [[130.0]], [[55.0]], [[174.0]], [[46.25]], [[155.0]], [[55.75]], [[59.25]], [[24.5]], [[56.0]], [[33.0]], [[43.0]], [[58.75]], [[65.0]], [[98.5]], [[43.75]], [[79.0]], [[91.0]], [[130.0]], [[55.25]], [[102.0]], [[23.5]], [[37.0]], [[58.0]], [[38.0]], [[88.5]], [[72.0]], [[41.0]], [[67.0]]]}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.41796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.5859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[70.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.37.feed_forward.shared_expert.down_proj": {"inputs": [[[121.5]]], "outputs": [[[21.25]], [[3.1635626299680998e+38]]], "params": {"weight": [[0.82421875]]}}, "language_model.model.layers.38.self_attn.qkv_proj": {"inputs": [[[47.25]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.38.self_attn.o_proj": {"inputs": [[[4.40625]]], "outputs": [[[26.75]], [[56.0]]], "params": {"weight": [[0.3984375]]}}, "language_model.model.layers.38.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.625]], [[12.8125]]]}, "language_model.model.layers.38.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.90625]]]}, "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.625]]]}, "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.625]]]}, "language_model.model.layers.38.self_attn.attn.impl.k_cache": {"inputs": [[[12.9375]]]}, "language_model.model.layers.38.self_attn.attn.impl.v_cache": {"inputs": [[[6.90625]]]}, "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[20.5]], [[12.9375]], [[6.90625]]], "outputs": [[[4.40625]], [[1.0]]]}, "language_model.model.layers.38.feed_forward.gate_up_proj": {"inputs": [[[31.125]]], "params": {"weight": [[0.447265625]]}}, "language_model.model.layers.38.feed_forward.down_proj": {"inputs": [[[109.0]]], "outputs": [[[24.75]], [[193.0]]], "params": {"weight": [[0.439453125]]}}, "language_model.model.layers.39.self_attn.qkv_proj": {"inputs": [[[49.75]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.39.self_attn.o_proj": {"inputs": [[[6.65625]]], "outputs": [[[129.0]], [[134.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.39.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8125]], [[14.4375]]]}, "language_model.model.layers.39.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.3125]]]}, "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8125]]]}, "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.65625]]]}, "language_model.model.layers.39.self_attn.attn.impl.k_cache": {"inputs": [[[14.4375]]]}, "language_model.model.layers.39.self_attn.attn.impl.v_cache": {"inputs": [[[8.3125]]]}, "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.125]], [[14.4375]], [[8.3125]]], "outputs": [[[6.25]], [[1.0]]]}, "language_model.model.layers.39.feed_forward.router": {"inputs": [[[61.5]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op": {"inputs": [[[61.5]]], "outputs": [[[11.375]], [[73.0]], [[58.0]], [[72.0]], [[72.0]], [[60.5]], [[81.5]], [[50.0]], [[114.0]], [[49.0]], [[50.75]], [[72.5]], [[32.0]], [[93.5]], [[144.0]], [[66.5]], [[44.25]], [[63.5]], [[140.0]], [[69.5]], [[55.25]], [[51.25]], [[81.5]], [[37.75]], [[50.25]], [[51.25]], [[112.0]], [[72.5]], [[79.0]], [[80.0]], [[69.0]], [[78.5]], [[156.0]], [[23.0]], [[47.75]], [[134.0]], [[81.5]], [[143.0]], [[91.5]], [[115.5]], [[100.0]], [[55.75]], [[51.25]], [[107.0]], [[75.0]], [[63.5]], [[35.0]], [[56.25]], [[54.25]], [[105.0]], [[55.0]], [[90.0]], [[81.5]], [[60.5]], [[49.25]], [[62.25]], [[70.5]], [[30.875]], [[106.0]], [[61.25]], [[70.0]], [[115.5]], [[98.0]], [[83.5]], [[32.5]], [[40.25]], [[72.5]], [[44.0]], [[97.0]], [[92.0]], [[187.0]], [[96.0]], [[27.875]], [[85.0]], [[101.5]], [[91.5]], [[81.0]], [[51.75]], [[112.0]], [[129.0]], [[37.5]], [[57.25]], [[72.0]], [[72.5]], [[46.75]], [[80.5]], [[108.5]], [[139.0]], [[44.0]], [[74.5]], [[89.5]], [[48.75]], [[80.5]], [[38.5]], [[55.25]], [[49.5]], [[42.0]], [[43.75]], [[101.0]], [[53.5]], [[34.5]], [[148.0]], [[53.5]], [[88.0]], [[76.0]], [[91.0]], [[133.0]], [[91.5]], [[117.0]], [[52.5]], [[64.0]], [[63.0]], [[67.5]], [[28.625]], [[68.5]], [[52.75]], [[110.5]], [[25.125]], [[33.0]], [[82.0]], [[81.5]], [[43.25]], [[91.0]], [[128.0]], [[32.5]], [[75.0]], [[28.125]], [[82.5]], [[78.0]]]}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[61.5]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.39.feed_forward.shared_expert.down_proj": {"inputs": [[[124.0]]], "outputs": [[[43.5]], [[3.1502703500102506e+38]]], "params": {"weight": [[0.7734375]]}}, "language_model.model.layers.40.self_attn.qkv_proj": {"inputs": [[[46.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.40.self_attn.o_proj": {"inputs": [[[10.8125]]], "outputs": [[[93.0]], [[90.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.40.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1484375]], [[14.75]]]}, "language_model.model.layers.40.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[13.5]]]}, "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1484375]]]}, "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[9.75]]]}, "language_model.model.layers.40.self_attn.attn.impl.k_cache": {"inputs": [[[14.75]]]}, "language_model.model.layers.40.self_attn.attn.impl.v_cache": {"inputs": [[[13.5]]]}, "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.875]], [[14.75]], [[13.1875]]], "outputs": [[[10.8125]], [[1.0]]]}, "language_model.model.layers.40.feed_forward.gate_up_proj": {"inputs": [[[40.75]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.40.feed_forward.down_proj": {"inputs": [[[199.0]]], "outputs": [[[22.625]], [[146.0]]], "params": {"weight": [[0.5234375]]}}, "language_model.model.layers.41.self_attn.qkv_proj": {"inputs": [[[33.75]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.41.self_attn.o_proj": {"inputs": [[[10.75]]], "outputs": [[[159.0]], [[164.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.41.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1796875]], [[15.0]]]}, "language_model.model.layers.41.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[19.0]]]}, "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1796875]]]}, "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[11.25]]]}, "language_model.model.layers.41.self_attn.attn.impl.k_cache": {"inputs": [[[15.0]]]}, "language_model.model.layers.41.self_attn.attn.impl.v_cache": {"inputs": [[[19.0]]]}, "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.75]], [[15.0]], [[19.0]]], "outputs": [[[10.625]], [[1.0]]]}, "language_model.model.layers.41.feed_forward.router": {"inputs": [[[68.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op": {"inputs": [[[68.0]]], "outputs": [[[17.5]], [[73.5]], [[27.25]], [[33.75]], [[70.0]], [[50.0]], [[50.5]], [[105.5]], [[35.5]], [[38.25]], [[111.5]], [[109.0]], [[117.0]], [[67.5]], [[70.5]], [[121.5]], [[95.5]], [[155.0]], [[76.0]], [[103.5]], [[48.25]], [[76.5]], [[40.75]], [[101.0]], [[87.5]], [[194.0]], [[152.0]], [[170.0]], [[77.5]], [[57.0]], [[90.0]], [[66.0]], [[75.5]], [[179.0]], [[63.0]], [[93.5]], [[112.0]], [[67.0]], [[121.0]], [[53.0]], [[60.75]], [[45.25]], [[140.0]], [[83.0]], [[45.0]], [[80.5]], [[99.5]], [[99.5]], [[97.5]], [[164.0]], [[141.0]], [[115.5]], [[77.5]], [[78.5]], [[83.5]], [[95.0]], [[92.5]], [[185.0]], [[77.0]], [[64.5]], [[74.0]], [[136.0]], [[84.0]], [[67.5]], [[74.5]], [[161.0]], [[73.0]], [[113.0]], [[98.0]], [[34.5]], [[170.0]], [[41.5]], [[44.0]], [[90.5]], [[28.875]], [[70.0]], [[125.0]], [[130.0]], [[52.25]], [[140.0]], [[91.5]], [[159.0]], [[102.0]], [[79.5]], [[77.5]], [[66.5]], [[50.25]], [[63.25]], [[48.5]], [[65.5]], [[96.0]], [[71.5]], [[160.0]], [[120.0]], [[59.25]], [[63.75]], [[66.0]], [[75.0]], [[104.0]], [[85.5]], [[68.5]], [[145.0]], [[54.5]], [[100.0]], [[54.75]], [[80.0]], [[58.5]], [[78.0]], [[102.5]], [[27.25]], [[80.5]], [[178.0]], [[96.0]], [[83.5]], [[79.0]], [[60.0]], [[167.0]], [[109.5]], [[149.0]], [[122.5]], [[86.5]], [[74.5]], [[83.0]], [[121.0]], [[43.5]], [[82.0]], [[50.25]], [[86.5]], [[175.0]]]}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.357421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[68.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.41.feed_forward.shared_expert.down_proj": {"inputs": [[[240.0]]], "outputs": [[[29.75]], [[3.1502703500102506e+38]]], "params": {"weight": [[0.6640625]]}}, "language_model.model.layers.42.self_attn.qkv_proj": {"inputs": [[[38.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.42.self_attn.o_proj": {"inputs": [[[12.8125]]], "outputs": [[[50.5]], [[134.0]]], "params": {"weight": [[0.447265625]]}}, "language_model.model.layers.42.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.234375]], [[23.25]]]}, "language_model.model.layers.42.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[15.25]]]}, "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.234375]]]}, "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[12.125]]]}, "language_model.model.layers.42.self_attn.attn.impl.k_cache": {"inputs": [[[23.25]]]}, "language_model.model.layers.42.self_attn.attn.impl.v_cache": {"inputs": [[[15.25]]]}, "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.25]], [[23.25]], [[15.25]]], "outputs": [[[12.8125]], [[1.0]]]}, "language_model.model.layers.42.feed_forward.gate_up_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.44140625]]}}, "language_model.model.layers.42.feed_forward.down_proj": {"inputs": [[[177.0]]], "outputs": [[[59.5]], [[338.0]]], "params": {"weight": [[0.796875]]}}, "language_model.model.layers.43.self_attn.qkv_proj": {"inputs": [[[45.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.43.self_attn.o_proj": {"inputs": [[[11.5625]]], "outputs": [[[94.5]], [[232.0]]], "params": {"weight": [[0.82421875]]}}, "language_model.model.layers.43.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.03125]], [[13.1875]]]}, "language_model.model.layers.43.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.75]]]}, "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.03125]]]}, "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[11.5625]]]}, "language_model.model.layers.43.self_attn.attn.impl.k_cache": {"inputs": [[[13.1875]]]}, "language_model.model.layers.43.self_attn.attn.impl.v_cache": {"inputs": [[[11.75]]]}, "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[17.5]], [[13.1875]], [[11.0]]], "outputs": [[[10.8125]], [[1.0]]]}, "language_model.model.layers.43.feed_forward.router": {"inputs": [[[68.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op": {"inputs": [[[68.0]]], "outputs": [[[23.875]], [[71.5]], [[158.0]], [[78.0]], [[96.5]], [[169.0]], [[59.0]], [[75.0]], [[127.0]], [[170.0]], [[115.0]], [[91.0]], [[115.5]], [[93.0]], [[88.0]], [[97.0]], [[122.0]], [[138.0]], [[73.5]], [[205.0]], [[52.25]], [[248.0]], [[101.0]], [[117.5]], [[146.0]], [[110.5]], [[82.5]], [[75.0]], [[75.0]], [[179.0]], [[71.0]], [[39.0]], [[87.0]], [[83.5]], [[136.0]], [[74.5]], [[139.0]], [[62.0]], [[111.0]], [[52.5]], [[92.0]], [[81.0]], [[169.0]], [[171.0]], [[68.5]], [[214.0]], [[127.0]], [[73.0]], [[35.0]], [[107.0]], [[73.5]], [[46.25]], [[174.0]], [[157.0]], [[81.0]], [[110.5]], [[128.0]], [[47.75]], [[39.5]], [[71.5]], [[90.5]], [[49.75]], [[57.0]], [[50.25]], [[149.0]], [[202.0]], [[47.5]], [[133.0]], [[80.0]], [[180.0]], [[55.0]], [[122.0]], [[74.5]], [[86.5]], [[62.75]], [[38.25]], [[119.5]], [[175.0]], [[70.0]], [[103.0]], [[116.0]], [[93.5]], [[103.0]], [[125.0]], [[82.5]], [[73.5]], [[171.0]], [[39.25]], [[126.0]], [[76.0]], [[96.0]], [[170.0]], [[58.25]], [[75.5]], [[95.5]], [[81.0]], [[64.5]], [[43.75]], [[187.0]], [[137.0]], [[60.0]], [[67.5]], [[81.5]], [[64.5]], [[70.0]], [[110.5]], [[109.5]], [[121.5]], [[73.0]], [[64.5]], [[71.5]], [[149.0]], [[104.0]], [[102.0]], [[72.5]], [[42.75]], [[109.0]], [[79.0]], [[93.5]], [[71.5]], [[71.0]], [[66.0]], [[134.0]], [[103.5]], [[132.0]], [[106.0]], [[171.0]], [[59.5]], [[79.5]]]}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.3828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[68.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.43.feed_forward.shared_expert.down_proj": {"inputs": [[[258.0]]], "outputs": [[[94.5]], [[3.1502703500102506e+38]]], "params": {"weight": [[0.72265625]]}}, "language_model.model.layers.44.self_attn.qkv_proj": {"inputs": [[[28.625]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.44.self_attn.o_proj": {"inputs": [[[16.25]]], "outputs": [[[68.0]], [[153.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.44.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.5390625]], [[14.125]]]}, "language_model.model.layers.44.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[18.375]]]}, "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.5390625]]]}, "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[12.0]]]}, "language_model.model.layers.44.self_attn.attn.impl.k_cache": {"inputs": [[[14.125]]]}, "language_model.model.layers.44.self_attn.attn.impl.v_cache": {"inputs": [[[18.375]]]}, "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[22.375]], [[14.125]], [[18.375]]], "outputs": [[[16.25]], [[1.0]]]}, "language_model.model.layers.44.feed_forward.gate_up_proj": {"inputs": [[[41.5]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.44.feed_forward.down_proj": {"inputs": [[[191.0]]], "outputs": [[[63.25]], [[191.0]]], "params": {"weight": [[0.76171875]]}}, "language_model.model.layers.45.self_attn.qkv_proj": {"inputs": [[[27.375]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.45.self_attn.o_proj": {"inputs": [[[19.875]]], "outputs": [[[169.0]], [[220.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.45.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.609375]], [[18.375]]]}, "language_model.model.layers.45.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[20.25]]]}, "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.609375]]]}, "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[11.6875]]]}, "language_model.model.layers.45.self_attn.attn.impl.k_cache": {"inputs": [[[18.375]]]}, "language_model.model.layers.45.self_attn.attn.impl.v_cache": {"inputs": [[[20.25]]]}, "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[19.0]], [[18.375]], [[20.25]]], "outputs": [[[19.875]], [[1.0]]]}, "language_model.model.layers.45.feed_forward.router": {"inputs": [[[51.75]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op": {"inputs": [[[51.75]]], "outputs": [[[15.0625]], [[26.875]], [[21.5]], [[59.25]], [[49.25]], [[0.0]], [[35.25]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[45.75]], [[25.875]], [[35.5]], [[44.0]], [[0.0]], [[0.0]], [[0.0]], [[48.0]], [[0.0]], [[0.0]], [[0.0]], [[37.0]], [[0.0]], [[23.75]], [[38.0]], [[21.375]], [[34.5]], [[38.0]], [[0.0]], [[20.25]], [[25.625]], [[45.25]], [[49.75]], [[51.0]], [[31.5]], [[0.0]], [[47.75]], [[16.375]], [[47.75]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[60.25]], [[0.0]], [[56.25]], [[0.0]], [[0.0]], [[0.0]], [[39.0]], [[0.0]], [[147.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[61.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[123.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[158.0]], [[24.625]], [[157.0]], [[42.25]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[55.75]], [[0.0]], [[28.75]], [[31.125]], [[70.5]], [[185.0]], [[0.0]], [[0.0]], [[24.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[10.25]], [[5.773376568485311e-18]], [[63.0]], [[0.0]], [[86.0]], [[0.0]], [[0.0]]]}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[51.75]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.45.feed_forward.shared_expert.down_proj": {"inputs": [[[326.0]]], "outputs": [[[44.5]], [[3.1502703500102506e+38]]], "params": {"weight": [[0.447265625]]}}, "language_model.model.layers.46.self_attn.qkv_proj": {"inputs": [[[26.75]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.46.self_attn.o_proj": {"inputs": [[[15.0625]]], "outputs": [[[101.0]], [[188.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.46.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.953125]], [[16.875]]]}, "language_model.model.layers.46.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[15.5625]]]}, "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.953125]]]}, "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[11.8125]]]}, "language_model.model.layers.46.self_attn.attn.impl.k_cache": {"inputs": [[[16.875]]]}, "language_model.model.layers.46.self_attn.attn.impl.v_cache": {"inputs": [[[15.5625]]]}, "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[25.125]], [[16.875]], [[15.5625]]], "outputs": [[[15.0625]], [[1.0]]]}, "language_model.model.layers.46.feed_forward.gate_up_proj": {"inputs": [[[40.5]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.46.feed_forward.down_proj": {"inputs": [[[512.0]]], "outputs": [[[1688.0]], [[2400.0]]], "params": {"weight": [[0.5703125]]}}, "language_model.model.layers.47.self_attn.qkv_proj": {"inputs": [[[19.875]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.47.self_attn.o_proj": {"inputs": [[[10.875]]], "outputs": [[[98.0]], [[126.5]]], "params": {"weight": [[0.419921875]]}}, "language_model.model.layers.47.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.37109375]], [[14.6875]]]}, "language_model.model.layers.47.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.5]]]}, "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.37109375]]]}, "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[10.8125]]]}, "language_model.model.layers.47.self_attn.attn.impl.k_cache": {"inputs": [[[14.6875]]]}, "language_model.model.layers.47.self_attn.attn.impl.v_cache": {"inputs": [[[11.5]]]}, "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[4.5625]], [[14.6875]], [[11.5]]], "outputs": [[[10.875]], [[1.0]]]}, "language_model.model.layers.47.feed_forward.router": {"inputs": [[[33.5]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op": {"inputs": [[[33.5]]], "outputs": [[[140.0]], [[57.0]], [[67.0]], [[56.75]], [[80.0]], [[76.0]], [[35.75]], [[52.5]], [[54.5]], [[98.0]], [[276.0]], [[60.25]], [[134.0]], [[89.5]], [[64.5]], [[148.0]], [[31.0]], [[137.0]], [[38.0]], [[59.0]], [[147.0]], [[105.5]], [[53.75]], [[81.5]], [[59.0]], [[231.0]], [[420.0]], [[27.75]], [[37.5]], [[46.25]], [[82.5]], [[48.5]], [[50.75]], [[23.0]], [[57.25]], [[107.5]], [[36.25]], [[29.375]], [[26.5]], [[149.0]], [[78.5]], [[48.25]], [[105.0]], [[126.0]], [[61.5]], [[119.0]], [[79.0]], [[123.0]], [[46.75]], [[21.375]], [[246.0]], [[156.0]], [[185.0]], [[122.5]], [[34.25]], [[48.0]], [[48.5]], [[113.5]], [[42.0]], [[30.5]], [[52.0]], [[62.5]], [[83.0]], [[85.5]], [[177.0]], [[89.0]], [[69.0]], [[38.75]], [[217.0]], [[31.625]], [[248.0]], [[139.0]], [[28.375]], [[26.0]], [[52.25]], [[55.25]], [[23.0]], [[39.25]], [[44.5]], [[95.5]], [[131.0]], [[107.5]], [[298.0]], [[736.0]], [[53.75]], [[34.0]], [[51.0]], [[79.0]], [[40.25]], [[26.25]], [[80.0]], [[155.0]], [[102.5]], [[115.0]], [[29.375]], [[32.75]], [[80.5]], [[59.75]], [[32.0]], [[31.25]], [[43.0]], [[80.5]], [[38.5]], [[82.5]], [[45.25]], [[45.5]], [[35.75]], [[43.75]], [[260.0]], [[33.5]], [[72.5]], [[91.5]], [[172.0]], [[43.75]], [[55.5]], [[76.0]], [[62.0]], [[62.5]], [[112.0]], [[47.0]], [[245.0]], [[150.0]], [[78.0]], [[64.5]], [[33.75]], [[40.75]], [[145.0]], [[123.5]], [[162.0]]]}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.79296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.4609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.5234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.462890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.66015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.65625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.39453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.70703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.7578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.46875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.4921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.61328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.69921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.6015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.380859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.5859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.380859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.6796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.5078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.46875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.78515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.8359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.69140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.45703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.5]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.3828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[1.015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.78515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.6328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.84375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.388671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.89453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.455078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.76953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.6640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.384765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.9296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.55078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.4375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.498046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.419921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.6484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.376953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.482421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.51171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.66796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.39453125]]}}, "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[33.5]]], "params": {"weight": [[0.46875]]}}, "language_model.model.layers.47.feed_forward.shared_expert.down_proj": {"inputs": [[[740.0]]], "outputs": [[[462.0]], [[3.1502703500102506e+38]]], "params": {"weight": [[0.419921875]]}}, "language_model.lm_head": {"inputs": [[[52.0]]], "params": {"weight": [[0.984375]]}}}} \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_4_8.npz b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_4_8.npz new file mode 100644 index 000000000000..a3787148f4b7 Binary files /dev/null and b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_4_8.npz differ diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_4_8_mod_list.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_4_8_mod_list.json new file mode 100644 index 000000000000..928cd9e2a2c5 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_4_8_mod_list.json @@ -0,0 +1,6935 @@ +[ + "vision_model.patch_embedding.linear", + "vision_model.model.layers.0.self_attn.qkv_proj", + "vision_model.model.layers.0.self_attn.o_proj", + "vision_model.model.layers.0.mlp.fc1", + "vision_model.model.layers.0.mlp.fc2", + "vision_model.model.layers.1.self_attn.qkv_proj", + "vision_model.model.layers.1.self_attn.o_proj", + "vision_model.model.layers.1.mlp.fc1", + "vision_model.model.layers.1.mlp.fc2", + "vision_model.model.layers.2.self_attn.qkv_proj", + "vision_model.model.layers.2.self_attn.o_proj", + "vision_model.model.layers.2.mlp.fc1", + "vision_model.model.layers.2.mlp.fc2", + "vision_model.model.layers.3.self_attn.qkv_proj", + "vision_model.model.layers.3.self_attn.o_proj", + "vision_model.model.layers.3.mlp.fc1", + "vision_model.model.layers.3.mlp.fc2", + "vision_model.model.layers.4.self_attn.qkv_proj", + "vision_model.model.layers.4.self_attn.o_proj", + "vision_model.model.layers.4.mlp.fc1", + "vision_model.model.layers.4.mlp.fc2", + "vision_model.model.layers.5.self_attn.qkv_proj", + "vision_model.model.layers.5.self_attn.o_proj", + "vision_model.model.layers.5.mlp.fc1", + "vision_model.model.layers.5.mlp.fc2", + "vision_model.model.layers.6.self_attn.qkv_proj", + "vision_model.model.layers.6.self_attn.o_proj", + "vision_model.model.layers.6.mlp.fc1", + "vision_model.model.layers.6.mlp.fc2", + "vision_model.model.layers.7.self_attn.qkv_proj", + "vision_model.model.layers.7.self_attn.o_proj", + "vision_model.model.layers.7.mlp.fc1", + "vision_model.model.layers.7.mlp.fc2", + "vision_model.model.layers.8.self_attn.qkv_proj", + "vision_model.model.layers.8.self_attn.o_proj", + "vision_model.model.layers.8.mlp.fc1", + "vision_model.model.layers.8.mlp.fc2", + "vision_model.model.layers.9.self_attn.qkv_proj", + "vision_model.model.layers.9.self_attn.o_proj", + "vision_model.model.layers.9.mlp.fc1", + "vision_model.model.layers.9.mlp.fc2", + "vision_model.model.layers.10.self_attn.qkv_proj", + "vision_model.model.layers.10.self_attn.o_proj", + "vision_model.model.layers.10.mlp.fc1", + "vision_model.model.layers.10.mlp.fc2", + "vision_model.model.layers.11.self_attn.qkv_proj", + "vision_model.model.layers.11.self_attn.o_proj", + "vision_model.model.layers.11.mlp.fc1", + "vision_model.model.layers.11.mlp.fc2", + "vision_model.model.layers.12.self_attn.qkv_proj", + "vision_model.model.layers.12.self_attn.o_proj", + "vision_model.model.layers.12.mlp.fc1", + "vision_model.model.layers.12.mlp.fc2", + "vision_model.model.layers.13.self_attn.qkv_proj", + "vision_model.model.layers.13.self_attn.o_proj", + "vision_model.model.layers.13.mlp.fc1", + "vision_model.model.layers.13.mlp.fc2", + "vision_model.model.layers.14.self_attn.qkv_proj", + "vision_model.model.layers.14.self_attn.o_proj", + "vision_model.model.layers.14.mlp.fc1", + "vision_model.model.layers.14.mlp.fc2", + "vision_model.model.layers.15.self_attn.qkv_proj", + "vision_model.model.layers.15.self_attn.o_proj", + "vision_model.model.layers.15.mlp.fc1", + "vision_model.model.layers.15.mlp.fc2", + "vision_model.model.layers.16.self_attn.qkv_proj", + "vision_model.model.layers.16.self_attn.o_proj", + "vision_model.model.layers.16.mlp.fc1", + "vision_model.model.layers.16.mlp.fc2", + "vision_model.model.layers.17.self_attn.qkv_proj", + "vision_model.model.layers.17.self_attn.o_proj", + "vision_model.model.layers.17.mlp.fc1", + "vision_model.model.layers.17.mlp.fc2", + "vision_model.model.layers.18.self_attn.qkv_proj", + "vision_model.model.layers.18.self_attn.o_proj", + "vision_model.model.layers.18.mlp.fc1", + "vision_model.model.layers.18.mlp.fc2", + "vision_model.model.layers.19.self_attn.qkv_proj", + "vision_model.model.layers.19.self_attn.o_proj", + "vision_model.model.layers.19.mlp.fc1", + "vision_model.model.layers.19.mlp.fc2", + "vision_model.model.layers.20.self_attn.qkv_proj", + "vision_model.model.layers.20.self_attn.o_proj", + "vision_model.model.layers.20.mlp.fc1", + "vision_model.model.layers.20.mlp.fc2", + "vision_model.model.layers.21.self_attn.qkv_proj", + "vision_model.model.layers.21.self_attn.o_proj", + "vision_model.model.layers.21.mlp.fc1", + "vision_model.model.layers.21.mlp.fc2", + "vision_model.model.layers.22.self_attn.qkv_proj", + "vision_model.model.layers.22.self_attn.o_proj", + "vision_model.model.layers.22.mlp.fc1", + "vision_model.model.layers.22.mlp.fc2", + "vision_model.model.layers.23.self_attn.qkv_proj", + "vision_model.model.layers.23.self_attn.o_proj", + "vision_model.model.layers.23.mlp.fc1", + "vision_model.model.layers.23.mlp.fc2", + "vision_model.model.layers.24.self_attn.qkv_proj", + "vision_model.model.layers.24.self_attn.o_proj", + "vision_model.model.layers.24.mlp.fc1", + "vision_model.model.layers.24.mlp.fc2", + "vision_model.model.layers.25.self_attn.qkv_proj", + "vision_model.model.layers.25.self_attn.o_proj", + "vision_model.model.layers.25.mlp.fc1", + "vision_model.model.layers.25.mlp.fc2", + "vision_model.model.layers.26.self_attn.qkv_proj", + "vision_model.model.layers.26.self_attn.o_proj", + "vision_model.model.layers.26.mlp.fc1", + "vision_model.model.layers.26.mlp.fc2", + "vision_model.model.layers.27.self_attn.qkv_proj", + "vision_model.model.layers.27.self_attn.o_proj", + "vision_model.model.layers.27.mlp.fc1", + "vision_model.model.layers.27.mlp.fc2", + "vision_model.model.layers.28.self_attn.qkv_proj", + "vision_model.model.layers.28.self_attn.o_proj", + "vision_model.model.layers.28.mlp.fc1", + "vision_model.model.layers.28.mlp.fc2", + "vision_model.model.layers.29.self_attn.qkv_proj", + "vision_model.model.layers.29.self_attn.o_proj", + "vision_model.model.layers.29.mlp.fc1", + "vision_model.model.layers.29.mlp.fc2", + "vision_model.model.layers.30.self_attn.qkv_proj", + "vision_model.model.layers.30.self_attn.o_proj", + "vision_model.model.layers.30.mlp.fc1", + "vision_model.model.layers.30.mlp.fc2", + "vision_model.model.layers.31.self_attn.qkv_proj", + "vision_model.model.layers.31.self_attn.o_proj", + "vision_model.model.layers.31.mlp.fc1", + "vision_model.model.layers.31.mlp.fc2", + "vision_model.model.layers.32.self_attn.qkv_proj", + "vision_model.model.layers.32.self_attn.o_proj", + "vision_model.model.layers.32.mlp.fc1", + "vision_model.model.layers.32.mlp.fc2", + "vision_model.model.layers.33.self_attn.qkv_proj", + "vision_model.model.layers.33.self_attn.o_proj", + "vision_model.model.layers.33.mlp.fc1", + "vision_model.model.layers.33.mlp.fc2", + "vision_model.vision_adapter.mlp.fc1", + "vision_model.vision_adapter.mlp.fc2", + "multi_modal_projector.linear_1", + "language_model.model.layers.0.self_attn.qkv_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.0.self_attn.attn.impl.softmax", + "language_model.model.layers.0.self_attn.attn.impl.matmul_av", + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.0.self_attn.attn.impl.k_cache", + "language_model.model.layers.0.self_attn.attn.impl.v_cache", + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.0.feed_forward.gate_up_proj", + "language_model.model.layers.0.feed_forward.down_proj", + "language_model.model.layers.1.self_attn.qkv_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.1.self_attn.attn.impl.softmax", + "language_model.model.layers.1.self_attn.attn.impl.matmul_av", + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.1.self_attn.attn.impl.k_cache", + "language_model.model.layers.1.self_attn.attn.impl.v_cache", + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.1.feed_forward.router", + "language_model.model.layers.1.feed_forward.experts", + "language_model.model.layers.1.feed_forward.experts.moe_op", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.1.feed_forward.shared_expert.down_proj", + "language_model.model.layers.2.self_attn.qkv_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.2.self_attn.attn.impl.softmax", + "language_model.model.layers.2.self_attn.attn.impl.matmul_av", + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.2.self_attn.attn.impl.k_cache", + "language_model.model.layers.2.self_attn.attn.impl.v_cache", + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.2.feed_forward.gate_up_proj", + "language_model.model.layers.2.feed_forward.down_proj", + "language_model.model.layers.3.self_attn.qkv_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.3.self_attn.attn.impl.softmax", + "language_model.model.layers.3.self_attn.attn.impl.matmul_av", + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.3.self_attn.attn.impl.k_cache", + "language_model.model.layers.3.self_attn.attn.impl.v_cache", + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.3.feed_forward.router", + "language_model.model.layers.3.feed_forward.experts", + "language_model.model.layers.3.feed_forward.experts.moe_op", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.3.feed_forward.shared_expert.down_proj", + "language_model.model.layers.4.self_attn.qkv_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.4.self_attn.attn.impl.softmax", + "language_model.model.layers.4.self_attn.attn.impl.matmul_av", + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.4.self_attn.attn.impl.k_cache", + "language_model.model.layers.4.self_attn.attn.impl.v_cache", + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.4.feed_forward.gate_up_proj", + "language_model.model.layers.4.feed_forward.down_proj", + "language_model.model.layers.5.self_attn.qkv_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.5.self_attn.attn.impl.softmax", + "language_model.model.layers.5.self_attn.attn.impl.matmul_av", + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.5.self_attn.attn.impl.k_cache", + "language_model.model.layers.5.self_attn.attn.impl.v_cache", + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.5.feed_forward.router", + "language_model.model.layers.5.feed_forward.experts", + "language_model.model.layers.5.feed_forward.experts.moe_op", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.5.feed_forward.shared_expert.down_proj", + "language_model.model.layers.6.self_attn.qkv_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.6.self_attn.attn.impl.softmax", + "language_model.model.layers.6.self_attn.attn.impl.matmul_av", + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.6.self_attn.attn.impl.k_cache", + "language_model.model.layers.6.self_attn.attn.impl.v_cache", + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.6.feed_forward.gate_up_proj", + "language_model.model.layers.6.feed_forward.down_proj", + "language_model.model.layers.7.self_attn.qkv_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.7.self_attn.attn.impl.softmax", + "language_model.model.layers.7.self_attn.attn.impl.matmul_av", + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.7.self_attn.attn.impl.k_cache", + "language_model.model.layers.7.self_attn.attn.impl.v_cache", + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.7.feed_forward.router", + "language_model.model.layers.7.feed_forward.experts", + "language_model.model.layers.7.feed_forward.experts.moe_op", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.7.feed_forward.shared_expert.down_proj", + "language_model.model.layers.8.self_attn.qkv_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.8.self_attn.attn.impl.softmax", + "language_model.model.layers.8.self_attn.attn.impl.matmul_av", + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.8.self_attn.attn.impl.k_cache", + "language_model.model.layers.8.self_attn.attn.impl.v_cache", + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.8.feed_forward.gate_up_proj", + "language_model.model.layers.8.feed_forward.down_proj", + "language_model.model.layers.9.self_attn.qkv_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.9.self_attn.attn.impl.softmax", + "language_model.model.layers.9.self_attn.attn.impl.matmul_av", + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.9.self_attn.attn.impl.k_cache", + "language_model.model.layers.9.self_attn.attn.impl.v_cache", + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.9.feed_forward.router", + "language_model.model.layers.9.feed_forward.experts", + "language_model.model.layers.9.feed_forward.experts.moe_op", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.9.feed_forward.shared_expert.down_proj", + "language_model.model.layers.10.self_attn.qkv_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.10.self_attn.attn.impl.softmax", + "language_model.model.layers.10.self_attn.attn.impl.matmul_av", + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.10.self_attn.attn.impl.k_cache", + "language_model.model.layers.10.self_attn.attn.impl.v_cache", + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.10.feed_forward.gate_up_proj", + "language_model.model.layers.10.feed_forward.down_proj", + "language_model.model.layers.11.self_attn.qkv_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.11.self_attn.attn.impl.softmax", + "language_model.model.layers.11.self_attn.attn.impl.matmul_av", + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.11.self_attn.attn.impl.k_cache", + "language_model.model.layers.11.self_attn.attn.impl.v_cache", + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.11.feed_forward.router", + "language_model.model.layers.11.feed_forward.experts", + "language_model.model.layers.11.feed_forward.experts.moe_op", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.11.feed_forward.shared_expert.down_proj", + "language_model.model.layers.12.self_attn.qkv_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.12.self_attn.attn.impl.softmax", + "language_model.model.layers.12.self_attn.attn.impl.matmul_av", + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.12.self_attn.attn.impl.k_cache", + "language_model.model.layers.12.self_attn.attn.impl.v_cache", + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.12.feed_forward.gate_up_proj", + "language_model.model.layers.12.feed_forward.down_proj", + "language_model.model.layers.13.self_attn.qkv_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.13.self_attn.attn.impl.softmax", + "language_model.model.layers.13.self_attn.attn.impl.matmul_av", + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.13.self_attn.attn.impl.k_cache", + "language_model.model.layers.13.self_attn.attn.impl.v_cache", + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.13.feed_forward.router", + "language_model.model.layers.13.feed_forward.experts", + "language_model.model.layers.13.feed_forward.experts.moe_op", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.13.feed_forward.shared_expert.down_proj", + "language_model.model.layers.14.self_attn.qkv_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.14.self_attn.attn.impl.softmax", + "language_model.model.layers.14.self_attn.attn.impl.matmul_av", + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.14.self_attn.attn.impl.k_cache", + "language_model.model.layers.14.self_attn.attn.impl.v_cache", + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.14.feed_forward.gate_up_proj", + "language_model.model.layers.14.feed_forward.down_proj", + "language_model.model.layers.15.self_attn.qkv_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.15.self_attn.attn.impl.softmax", + "language_model.model.layers.15.self_attn.attn.impl.matmul_av", + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.15.self_attn.attn.impl.k_cache", + "language_model.model.layers.15.self_attn.attn.impl.v_cache", + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.15.feed_forward.router", + "language_model.model.layers.15.feed_forward.experts", + "language_model.model.layers.15.feed_forward.experts.moe_op", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.15.feed_forward.shared_expert.down_proj", + "language_model.model.layers.16.self_attn.qkv_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.16.self_attn.attn.impl.softmax", + "language_model.model.layers.16.self_attn.attn.impl.matmul_av", + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.16.self_attn.attn.impl.k_cache", + "language_model.model.layers.16.self_attn.attn.impl.v_cache", + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.16.feed_forward.gate_up_proj", + "language_model.model.layers.16.feed_forward.down_proj", + "language_model.model.layers.17.self_attn.qkv_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.17.self_attn.attn.impl.softmax", + "language_model.model.layers.17.self_attn.attn.impl.matmul_av", + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.17.self_attn.attn.impl.k_cache", + "language_model.model.layers.17.self_attn.attn.impl.v_cache", + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.17.feed_forward.router", + "language_model.model.layers.17.feed_forward.experts", + "language_model.model.layers.17.feed_forward.experts.moe_op", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.17.feed_forward.shared_expert.down_proj", + "language_model.model.layers.18.self_attn.qkv_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.18.self_attn.attn.impl.softmax", + "language_model.model.layers.18.self_attn.attn.impl.matmul_av", + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.18.self_attn.attn.impl.k_cache", + "language_model.model.layers.18.self_attn.attn.impl.v_cache", + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.18.feed_forward.gate_up_proj", + "language_model.model.layers.18.feed_forward.down_proj", + "language_model.model.layers.19.self_attn.qkv_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.19.self_attn.attn.impl.softmax", + "language_model.model.layers.19.self_attn.attn.impl.matmul_av", + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.19.self_attn.attn.impl.k_cache", + "language_model.model.layers.19.self_attn.attn.impl.v_cache", + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.19.feed_forward.router", + "language_model.model.layers.19.feed_forward.experts", + "language_model.model.layers.19.feed_forward.experts.moe_op", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.19.feed_forward.shared_expert.down_proj", + "language_model.model.layers.20.self_attn.qkv_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.20.self_attn.attn.impl.softmax", + "language_model.model.layers.20.self_attn.attn.impl.matmul_av", + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.20.self_attn.attn.impl.k_cache", + "language_model.model.layers.20.self_attn.attn.impl.v_cache", + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.20.feed_forward.gate_up_proj", + "language_model.model.layers.20.feed_forward.down_proj", + "language_model.model.layers.21.self_attn.qkv_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.21.self_attn.attn.impl.softmax", + "language_model.model.layers.21.self_attn.attn.impl.matmul_av", + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.21.self_attn.attn.impl.k_cache", + "language_model.model.layers.21.self_attn.attn.impl.v_cache", + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.21.feed_forward.router", + "language_model.model.layers.21.feed_forward.experts", + "language_model.model.layers.21.feed_forward.experts.moe_op", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.21.feed_forward.shared_expert.down_proj", + "language_model.model.layers.22.self_attn.qkv_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.22.self_attn.attn.impl.softmax", + "language_model.model.layers.22.self_attn.attn.impl.matmul_av", + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.22.self_attn.attn.impl.k_cache", + "language_model.model.layers.22.self_attn.attn.impl.v_cache", + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.22.feed_forward.gate_up_proj", + "language_model.model.layers.22.feed_forward.down_proj", + "language_model.model.layers.23.self_attn.qkv_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.23.self_attn.attn.impl.softmax", + "language_model.model.layers.23.self_attn.attn.impl.matmul_av", + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.23.self_attn.attn.impl.k_cache", + "language_model.model.layers.23.self_attn.attn.impl.v_cache", + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.23.feed_forward.router", + "language_model.model.layers.23.feed_forward.experts", + "language_model.model.layers.23.feed_forward.experts.moe_op", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.23.feed_forward.shared_expert.down_proj", + "language_model.model.layers.24.self_attn.qkv_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.24.self_attn.attn.impl.softmax", + "language_model.model.layers.24.self_attn.attn.impl.matmul_av", + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.24.self_attn.attn.impl.k_cache", + "language_model.model.layers.24.self_attn.attn.impl.v_cache", + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.24.feed_forward.gate_up_proj", + "language_model.model.layers.24.feed_forward.down_proj", + "language_model.model.layers.25.self_attn.qkv_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.25.self_attn.attn.impl.softmax", + "language_model.model.layers.25.self_attn.attn.impl.matmul_av", + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.25.self_attn.attn.impl.k_cache", + "language_model.model.layers.25.self_attn.attn.impl.v_cache", + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.25.feed_forward.router", + "language_model.model.layers.25.feed_forward.experts", + "language_model.model.layers.25.feed_forward.experts.moe_op", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.25.feed_forward.shared_expert.down_proj", + "language_model.model.layers.26.self_attn.qkv_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.26.self_attn.attn.impl.softmax", + "language_model.model.layers.26.self_attn.attn.impl.matmul_av", + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.26.self_attn.attn.impl.k_cache", + "language_model.model.layers.26.self_attn.attn.impl.v_cache", + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.26.feed_forward.gate_up_proj", + "language_model.model.layers.26.feed_forward.down_proj", + "language_model.model.layers.27.self_attn.qkv_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.27.self_attn.attn.impl.softmax", + "language_model.model.layers.27.self_attn.attn.impl.matmul_av", + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.27.self_attn.attn.impl.k_cache", + "language_model.model.layers.27.self_attn.attn.impl.v_cache", + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.27.feed_forward.router", + "language_model.model.layers.27.feed_forward.experts", + "language_model.model.layers.27.feed_forward.experts.moe_op", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.27.feed_forward.shared_expert.down_proj", + "language_model.model.layers.28.self_attn.qkv_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.28.self_attn.attn.impl.softmax", + "language_model.model.layers.28.self_attn.attn.impl.matmul_av", + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.28.self_attn.attn.impl.k_cache", + "language_model.model.layers.28.self_attn.attn.impl.v_cache", + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.28.feed_forward.gate_up_proj", + "language_model.model.layers.28.feed_forward.down_proj", + "language_model.model.layers.29.self_attn.qkv_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.29.self_attn.attn.impl.softmax", + "language_model.model.layers.29.self_attn.attn.impl.matmul_av", + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.29.self_attn.attn.impl.k_cache", + "language_model.model.layers.29.self_attn.attn.impl.v_cache", + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.29.feed_forward.router", + "language_model.model.layers.29.feed_forward.experts", + "language_model.model.layers.29.feed_forward.experts.moe_op", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.29.feed_forward.shared_expert.down_proj", + "language_model.model.layers.30.self_attn.qkv_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.30.self_attn.attn.impl.softmax", + "language_model.model.layers.30.self_attn.attn.impl.matmul_av", + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.30.self_attn.attn.impl.k_cache", + "language_model.model.layers.30.self_attn.attn.impl.v_cache", + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.30.feed_forward.gate_up_proj", + "language_model.model.layers.30.feed_forward.down_proj", + "language_model.model.layers.31.self_attn.qkv_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.31.self_attn.attn.impl.softmax", + "language_model.model.layers.31.self_attn.attn.impl.matmul_av", + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.31.self_attn.attn.impl.k_cache", + "language_model.model.layers.31.self_attn.attn.impl.v_cache", + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.31.feed_forward.router", + "language_model.model.layers.31.feed_forward.experts", + "language_model.model.layers.31.feed_forward.experts.moe_op", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.31.feed_forward.shared_expert.down_proj", + "language_model.model.layers.32.self_attn.qkv_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.32.self_attn.attn.impl.softmax", + "language_model.model.layers.32.self_attn.attn.impl.matmul_av", + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.32.self_attn.attn.impl.k_cache", + "language_model.model.layers.32.self_attn.attn.impl.v_cache", + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.32.feed_forward.gate_up_proj", + "language_model.model.layers.32.feed_forward.down_proj", + "language_model.model.layers.33.self_attn.qkv_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.33.self_attn.attn.impl.softmax", + "language_model.model.layers.33.self_attn.attn.impl.matmul_av", + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.33.self_attn.attn.impl.k_cache", + "language_model.model.layers.33.self_attn.attn.impl.v_cache", + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.33.feed_forward.router", + "language_model.model.layers.33.feed_forward.experts", + "language_model.model.layers.33.feed_forward.experts.moe_op", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.33.feed_forward.shared_expert.down_proj", + "language_model.model.layers.34.self_attn.qkv_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.34.self_attn.attn.impl.softmax", + "language_model.model.layers.34.self_attn.attn.impl.matmul_av", + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.34.self_attn.attn.impl.k_cache", + "language_model.model.layers.34.self_attn.attn.impl.v_cache", + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.34.feed_forward.gate_up_proj", + "language_model.model.layers.34.feed_forward.down_proj", + "language_model.model.layers.35.self_attn.qkv_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.35.self_attn.attn.impl.softmax", + "language_model.model.layers.35.self_attn.attn.impl.matmul_av", + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.35.self_attn.attn.impl.k_cache", + "language_model.model.layers.35.self_attn.attn.impl.v_cache", + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.35.feed_forward.router", + "language_model.model.layers.35.feed_forward.experts", + "language_model.model.layers.35.feed_forward.experts.moe_op", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.35.feed_forward.shared_expert.down_proj", + "language_model.model.layers.36.self_attn.qkv_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.36.self_attn.attn.impl.softmax", + "language_model.model.layers.36.self_attn.attn.impl.matmul_av", + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.36.self_attn.attn.impl.k_cache", + "language_model.model.layers.36.self_attn.attn.impl.v_cache", + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.36.feed_forward.gate_up_proj", + "language_model.model.layers.36.feed_forward.down_proj", + "language_model.model.layers.37.self_attn.qkv_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.37.self_attn.attn.impl.softmax", + "language_model.model.layers.37.self_attn.attn.impl.matmul_av", + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.37.self_attn.attn.impl.k_cache", + "language_model.model.layers.37.self_attn.attn.impl.v_cache", + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.37.feed_forward.router", + "language_model.model.layers.37.feed_forward.experts", + "language_model.model.layers.37.feed_forward.experts.moe_op", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.37.feed_forward.shared_expert.down_proj", + "language_model.model.layers.38.self_attn.qkv_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.38.self_attn.attn.impl.softmax", + "language_model.model.layers.38.self_attn.attn.impl.matmul_av", + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.38.self_attn.attn.impl.k_cache", + "language_model.model.layers.38.self_attn.attn.impl.v_cache", + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.38.feed_forward.gate_up_proj", + "language_model.model.layers.38.feed_forward.down_proj", + "language_model.model.layers.39.self_attn.qkv_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.39.self_attn.attn.impl.softmax", + "language_model.model.layers.39.self_attn.attn.impl.matmul_av", + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.39.self_attn.attn.impl.k_cache", + "language_model.model.layers.39.self_attn.attn.impl.v_cache", + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.39.feed_forward.router", + "language_model.model.layers.39.feed_forward.experts", + "language_model.model.layers.39.feed_forward.experts.moe_op", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.39.feed_forward.shared_expert.down_proj", + "language_model.model.layers.40.self_attn.qkv_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.40.self_attn.attn.impl.softmax", + "language_model.model.layers.40.self_attn.attn.impl.matmul_av", + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.40.self_attn.attn.impl.k_cache", + "language_model.model.layers.40.self_attn.attn.impl.v_cache", + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.40.feed_forward.gate_up_proj", + "language_model.model.layers.40.feed_forward.down_proj", + "language_model.model.layers.41.self_attn.qkv_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.41.self_attn.attn.impl.softmax", + "language_model.model.layers.41.self_attn.attn.impl.matmul_av", + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.41.self_attn.attn.impl.k_cache", + "language_model.model.layers.41.self_attn.attn.impl.v_cache", + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.41.feed_forward.router", + "language_model.model.layers.41.feed_forward.experts", + "language_model.model.layers.41.feed_forward.experts.moe_op", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.41.feed_forward.shared_expert.down_proj", + "language_model.model.layers.42.self_attn.qkv_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.42.self_attn.attn.impl.softmax", + "language_model.model.layers.42.self_attn.attn.impl.matmul_av", + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.42.self_attn.attn.impl.k_cache", + "language_model.model.layers.42.self_attn.attn.impl.v_cache", + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.42.feed_forward.gate_up_proj", + "language_model.model.layers.42.feed_forward.down_proj", + "language_model.model.layers.43.self_attn.qkv_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.43.self_attn.attn.impl.softmax", + "language_model.model.layers.43.self_attn.attn.impl.matmul_av", + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.43.self_attn.attn.impl.k_cache", + "language_model.model.layers.43.self_attn.attn.impl.v_cache", + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.43.feed_forward.router", + "language_model.model.layers.43.feed_forward.experts", + "language_model.model.layers.43.feed_forward.experts.moe_op", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.43.feed_forward.shared_expert.down_proj", + "language_model.model.layers.44.self_attn.qkv_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.44.self_attn.attn.impl.softmax", + "language_model.model.layers.44.self_attn.attn.impl.matmul_av", + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.44.self_attn.attn.impl.k_cache", + "language_model.model.layers.44.self_attn.attn.impl.v_cache", + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.44.feed_forward.gate_up_proj", + "language_model.model.layers.44.feed_forward.down_proj", + "language_model.model.layers.45.self_attn.qkv_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.45.self_attn.attn.impl.softmax", + "language_model.model.layers.45.self_attn.attn.impl.matmul_av", + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.45.self_attn.attn.impl.k_cache", + "language_model.model.layers.45.self_attn.attn.impl.v_cache", + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.45.feed_forward.router", + "language_model.model.layers.45.feed_forward.experts", + "language_model.model.layers.45.feed_forward.experts.moe_op", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.45.feed_forward.shared_expert.down_proj", + "language_model.model.layers.46.self_attn.qkv_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.46.self_attn.attn.impl.softmax", + "language_model.model.layers.46.self_attn.attn.impl.matmul_av", + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.46.self_attn.attn.impl.k_cache", + "language_model.model.layers.46.self_attn.attn.impl.v_cache", + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.46.feed_forward.gate_up_proj", + "language_model.model.layers.46.feed_forward.down_proj", + "language_model.model.layers.47.self_attn.qkv_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.47.self_attn.attn.impl.softmax", + "language_model.model.layers.47.self_attn.attn.impl.matmul_av", + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.47.self_attn.attn.impl.k_cache", + "language_model.model.layers.47.self_attn.attn.impl.v_cache", + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.47.feed_forward.router", + "language_model.model.layers.47.feed_forward.experts", + "language_model.model.layers.47.feed_forward.experts.moe_op", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.47.feed_forward.shared_expert.down_proj", + "language_model.lm_head" +] \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_5_8.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_5_8.json new file mode 100644 index 000000000000..9427ae3ecc59 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_5_8.json @@ -0,0 +1 @@ +{"GlobalRank": null, "LocalRank": 5, "Mode": "DynamicRange", "Nodes": {"language_model.model.layers.0.self_attn.qkv_proj": {"inputs": [[[11.1875]]], "params": {"weight": [[0.3671875]]}}, "language_model.model.layers.0.self_attn.o_proj": {"inputs": [[[0.462890625]]], "outputs": [[[0.80078125]], [[3.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.0.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.365234375]], [[6.3125]]]}, "language_model.model.layers.0.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.5546875]]]}, "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.365234375]]]}, "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.5546875]]]}, "language_model.model.layers.0.self_attn.attn.impl.k_cache": {"inputs": [[[6.3125]]]}, "language_model.model.layers.0.self_attn.attn.impl.v_cache": {"inputs": [[[0.5546875]]]}, "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[4.3125]], [[6.3125]], [[0.5546875]]], "outputs": [[[0.462890625]], [[1.0]]]}, "language_model.model.layers.0.feed_forward.gate_up_proj": {"inputs": [[[2.546875]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.0.feed_forward.down_proj": {"inputs": [[[3.84375]]], "outputs": [[[26.75]], [[65.0]]], "params": {"weight": [[0.7421875]]}}, "language_model.model.layers.1.self_attn.qkv_proj": {"inputs": [[[12.3125]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.1.self_attn.o_proj": {"inputs": [[[0.76171875]]], "outputs": [[[0.52734375]], [[1.0078125]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.1.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.78515625]], [[9.625]]]}, "language_model.model.layers.1.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.890625]]]}, "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.78515625]]]}, "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.7421875]]]}, "language_model.model.layers.1.self_attn.attn.impl.k_cache": {"inputs": [[[9.625]]]}, "language_model.model.layers.1.self_attn.attn.impl.v_cache": {"inputs": [[[0.890625]]]}, "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.125]], [[8.5625]], [[0.890625]]], "outputs": [[[0.76171875]], [[1.0]]]}, "language_model.model.layers.1.feed_forward.router": {"inputs": [[[1.8828125]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op": {"inputs": [[[1.8828125]]], "outputs": [[[1.265625]], [[12.0]], [[0.0]], [[0.8828125]], [[12.5625]], [[9.4375]], [[13.5625]], [[13.0]], [[9.3125]], [[7.3125]], [[14.75]], [[10.9375]], [[14.0]], [[4.03125]], [[13.75]], [[9.9375]], [[20.25]], [[10.75]], [[4.46875]], [[8.625]], [[4.84375]], [[14.25]], [[14.0]], [[0.0]], [[14.1875]], [[7.90625]], [[2.015625]], [[9.125]], [[9.3125]], [[4.625]], [[9.0]], [[8.125]], [[18.125]], [[0.197265625]], [[3.984375]], [[0.0]], [[14.5625]], [[1.65625]], [[12.1875]], [[4.84375]], [[12.1875]], [[12.125]], [[11.5]], [[11.375]], [[9.25]], [[5.21875]], [[13.25]], [[11.5625]], [[8.9375]], [[3.71875]], [[0.0]], [[9.8125]], [[12.25]], [[8.0625]], [[3.96875]], [[8.9375]], [[17.125]], [[10.125]], [[1.046875]], [[7.875]], [[9.25]], [[10.3125]], [[11.4375]], [[17.625]], [[0.0]], [[10.125]], [[9.8125]], [[7.9375]], [[8.6875]], [[3.65625]], [[4.625]], [[11.8125]], [[7.40625]], [[12.1875]], [[15.1875]], [[11.625]], [[5.34375]], [[10.875]], [[10.125]], [[4.3125]], [[8.8125]], [[0.0]], [[1.2890625]], [[18.125]], [[2.359375]], [[14.0]], [[15.5]], [[9.8125]], [[14.375]], [[7.15625]], [[10.75]], [[9.8125]], [[9.4375]], [[12.625]], [[0.0]], [[12.5]], [[11.125]], [[0.11572265625]], [[0.0]], [[13.0625]], [[11.25]], [[15.5]], [[11.0]], [[11.0625]], [[13.8125]], [[1.234375]], [[12.375]], [[0.07568359375]], [[0.75]], [[4.1875]], [[0.1611328125]], [[8.0625]], [[2.421875]], [[13.375]], [[0.0]], [[13.375]], [[13.875]], [[14.0]], [[7.71875]], [[15.875]], [[5.96875]], [[11.75]], [[11.8125]], [[5.34375]], [[6.09375]], [[10.5]], [[7.875]], [[12.0625]], [[13.0]]]}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.43359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.35546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.072265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.38671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.5625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[1.8828125]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.1.feed_forward.shared_expert.down_proj": {"inputs": [[[4.28125]]], "outputs": [[[0.310546875]], [[3.176854909925949e+38]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.2.self_attn.qkv_proj": {"inputs": [[[9.875]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.2.self_attn.o_proj": {"inputs": [[[0.79296875]]], "outputs": [[[0.6875]], [[1.328125]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.2.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.60546875]], [[9.5]]]}, "language_model.model.layers.2.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.296875]]]}, "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.60546875]]]}, "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.890625]]]}, "language_model.model.layers.2.self_attn.attn.impl.k_cache": {"inputs": [[[9.5]]]}, "language_model.model.layers.2.self_attn.attn.impl.v_cache": {"inputs": [[[1.296875]]]}, "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[7.59375]], [[9.5]], [[1.296875]]], "outputs": [[[0.79296875]], [[1.0]]]}, "language_model.model.layers.2.feed_forward.gate_up_proj": {"inputs": [[[6.125]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.2.feed_forward.down_proj": {"inputs": [[[14.375]]], "outputs": [[[4.75]], [[25.25]]], "params": {"weight": [[0.55078125]]}}, "language_model.model.layers.3.self_attn.qkv_proj": {"inputs": [[[7.21875]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.3.self_attn.o_proj": {"inputs": [[[0.56640625]]], "outputs": [[[0.59765625]], [[2.40625]]], "params": {"weight": [[0.408203125]]}}, "language_model.model.layers.3.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.126953125]], [[3.09375]]]}, "language_model.model.layers.3.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.484375]]]}, "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.126953125]]]}, "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.4609375]]]}, "language_model.model.layers.3.self_attn.attn.impl.k_cache": {"inputs": [[[3.09375]]]}, "language_model.model.layers.3.self_attn.attn.impl.v_cache": {"inputs": [[[0.484375]]]}, "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[1.6796875]], [[3.09375]], [[0.57421875]]], "outputs": [[[0.56640625]], [[1.0]]]}, "language_model.model.layers.3.feed_forward.router": {"inputs": [[[8.75]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op": {"inputs": [[[8.75]]], "outputs": [[[198.0]], [[0.671875]], [[9.8125]], [[15.1875]], [[13.0625]], [[0.0]], [[7.0625]], [[16.75]], [[19.125]], [[12.375]], [[15.5625]], [[12.3125]], [[11.1875]], [[15.9375]], [[21.25]], [[9.0625]], [[19.5]], [[0.796875]], [[6.5]], [[13.125]], [[19.125]], [[6.21875]], [[14.25]], [[17.0]], [[14.25]], [[8.0625]], [[6.40625]], [[12.125]], [[6.15625]], [[8.5]], [[8.75]], [[15.3125]], [[8.4375]], [[13.375]], [[6.9375]], [[11.625]], [[13.8125]], [[3.234375]], [[0.68359375]], [[12.3125]], [[8.1875]], [[13.9375]], [[14.3125]], [[4.8125]], [[9.1875]], [[12.4375]], [[3.015625]], [[12.75]], [[10.875]], [[4.9375]], [[16.75]], [[11.8125]], [[14.0625]], [[14.0]], [[1.796875]], [[6.65625]], [[12.5625]], [[3.640625]], [[11.4375]], [[17.25]], [[9.75]], [[4.875]], [[18.875]], [[15.375]], [[0.1875]], [[5.125]], [[10.4375]], [[17.625]], [[0.50390625]], [[5.4375]], [[0.0]], [[10.8125]], [[14.4375]], [[15.9375]], [[5.9375]], [[4.46875]], [[8.5]], [[9.8125]], [[0.1279296875]], [[16.125]], [[9.4375]], [[11.5625]], [[0.0]], [[4.625]], [[21.25]], [[7.1875]], [[1.4375]], [[7.78125]], [[2.265625]], [[16.875]], [[14.875]], [[19.625]], [[4.1875]], [[5.15625]], [[9.5625]], [[0.0]], [[11.75]], [[13.625]], [[9.9375]], [[15.1875]], [[1.0078125]], [[13.8125]], [[0.765625]], [[10.125]], [[16.875]], [[0.0]], [[3.0625]], [[11.3125]], [[14.75]], [[15.0625]], [[1.5]], [[13.3125]], [[3.5]], [[10.375]], [[2.28125]], [[1.953125]], [[2.171875]], [[8.25]], [[1.0625]], [[0.93359375]], [[2.703125]], [[18.875]], [[7.875]], [[6.78125]], [[11.5]], [[3.25]], [[13.625]], [[9.9375]], [[11.9375]]]}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.75]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[8.75]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.3.feed_forward.shared_expert.down_proj": {"inputs": [[[4.75]]], "outputs": [[[0.326171875]], [[8.706443372391199e+37]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.4.self_attn.qkv_proj": {"inputs": [[[13.75]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.4.self_attn.o_proj": {"inputs": [[[1.5]]], "outputs": [[[1.015625]], [[2.046875]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.4.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.69140625]], [[11.375]]]}, "language_model.model.layers.4.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.890625]]]}, "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.69140625]]]}, "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.7734375]]]}, "language_model.model.layers.4.self_attn.attn.impl.k_cache": {"inputs": [[[11.375]]]}, "language_model.model.layers.4.self_attn.attn.impl.v_cache": {"inputs": [[[2.890625]]]}, "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[7.9375]], [[11.375]], [[2.890625]]], "outputs": [[[1.5]], [[1.0]]]}, "language_model.model.layers.4.feed_forward.gate_up_proj": {"inputs": [[[7.09375]]], "params": {"weight": [[0.32421875]]}}, "language_model.model.layers.4.feed_forward.down_proj": {"inputs": [[[7.78125]]], "outputs": [[[2.796875]], [[4.96875]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.5.self_attn.qkv_proj": {"inputs": [[[17.375]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.5.self_attn.o_proj": {"inputs": [[[1.6640625]]], "outputs": [[[0.97265625]], [[2.21875]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.5.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.65234375]], [[9.5]]]}, "language_model.model.layers.5.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.625]]]}, "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.65234375]]]}, "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.078125]]]}, "language_model.model.layers.5.self_attn.attn.impl.k_cache": {"inputs": [[[9.5]]]}, "language_model.model.layers.5.self_attn.attn.impl.v_cache": {"inputs": [[[3.625]]]}, "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.25]], [[9.5]], [[3.625]]], "outputs": [[[1.6640625]], [[1.0]]]}, "language_model.model.layers.5.feed_forward.router": {"inputs": [[[19.25]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op": {"inputs": [[[19.25]]], "outputs": [[[64.5]], [[16.125]], [[3.171875]], [[6.125]], [[18.0]], [[11.6875]], [[9.5625]], [[11.0]], [[23.125]], [[11.125]], [[25.25]], [[18.0]], [[14.25]], [[17.125]], [[5.8125]], [[10.125]], [[13.9375]], [[11.9375]], [[10.5]], [[11.5]], [[5.8125]], [[20.75]], [[12.0625]], [[5.03125]], [[16.875]], [[3.0625]], [[1.046875]], [[14.125]], [[31.25]], [[7.125]], [[31.25]], [[25.375]], [[12.125]], [[12.0]], [[22.75]], [[16.125]], [[11.8125]], [[12.3125]], [[7.25]], [[14.125]], [[16.125]], [[13.375]], [[4.59375]], [[9.875]], [[11.3125]], [[12.5]], [[12.9375]], [[8.0625]], [[16.75]], [[9.4375]], [[12.6875]], [[16.375]], [[15.75]], [[0.6328125]], [[7.625]], [[6.71875]], [[2.96875]], [[11.1875]], [[16.375]], [[25.625]], [[0.68359375]], [[20.375]], [[18.875]], [[9.75]], [[22.5]], [[8.3125]], [[9.375]], [[6.59375]], [[8.0]], [[20.125]], [[4.90625]], [[5.0625]], [[20.125]], [[17.75]], [[16.5]], [[4.96875]], [[14.3125]], [[17.875]], [[6.375]], [[14.0625]], [[25.875]], [[2.890625]], [[9.875]], [[6.40625]], [[11.3125]], [[5.125]], [[12.8125]], [[0.19921875]], [[16.625]], [[14.875]], [[23.625]], [[17.0]], [[8.5625]], [[14.4375]], [[15.75]], [[18.75]], [[2.375]], [[21.75]], [[11.3125]], [[6.0]], [[13.6875]], [[7.90625]], [[3.703125]], [[2.15625]], [[24.125]], [[25.5]], [[15.75]], [[5.96875]], [[17.125]], [[11.1875]], [[11.8125]], [[8.9375]], [[9.8125]], [[5.90625]], [[6.0]], [[11.0]], [[13.75]], [[3.328125]], [[9.9375]], [[7.0625]], [[9.9375]], [[17.5]], [[2.953125]], [[11.6875]], [[17.875]], [[16.0]], [[5.78125]], [[9.4375]], [[4.9375]]]}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.396484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.58984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[19.25]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.5.feed_forward.shared_expert.down_proj": {"inputs": [[[7.90625]]], "outputs": [[[0.578125]], [[2.9110093107689658e+38]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.6.self_attn.qkv_proj": {"inputs": [[[12.9375]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.6.self_attn.o_proj": {"inputs": [[[1.4453125]]], "outputs": [[[0.66796875]], [[1.5234375]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.6.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8828125]], [[14.5]]]}, "language_model.model.layers.6.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.859375]]]}, "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8828125]]]}, "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.609375]]]}, "language_model.model.layers.6.self_attn.attn.impl.k_cache": {"inputs": [[[14.5]]]}, "language_model.model.layers.6.self_attn.attn.impl.v_cache": {"inputs": [[[2.859375]]]}, "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.5625]], [[14.5]], [[2.859375]]], "outputs": [[[1.4453125]], [[1.0]]]}, "language_model.model.layers.6.feed_forward.gate_up_proj": {"inputs": [[[7.71875]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.6.feed_forward.down_proj": {"inputs": [[[7.78125]]], "outputs": [[[2.328125]], [[11.1875]]], "params": {"weight": [[0.474609375]]}}, "language_model.model.layers.7.self_attn.qkv_proj": {"inputs": [[[7.09375]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.7.self_attn.o_proj": {"inputs": [[[0.56640625]]], "outputs": [[[1.84375]], [[3.84375]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.7.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.265625]], [[2.5]]]}, "language_model.model.layers.7.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.03125]]]}, "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.265625]]]}, "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.63671875]]]}, "language_model.model.layers.7.self_attn.attn.impl.k_cache": {"inputs": [[[2.5]]]}, "language_model.model.layers.7.self_attn.attn.impl.v_cache": {"inputs": [[[2.03125]]]}, "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[3.71875]], [[2.5]], [[2.03125]]], "outputs": [[[0.56640625]], [[1.0]]]}, "language_model.model.layers.7.feed_forward.router": {"inputs": [[[11.6875]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op": {"inputs": [[[11.6875]]], "outputs": [[[2.53125]], [[7.59375]], [[14.75]], [[23.0]], [[12.8125]], [[6.65625]], [[18.5]], [[14.5625]], [[25.0]], [[8.6875]], [[13.25]], [[24.625]], [[7.625]], [[11.25]], [[25.75]], [[4.0625]], [[2.34375]], [[17.5]], [[12.5]], [[32.75]], [[15.5]], [[10.9375]], [[25.375]], [[24.375]], [[19.25]], [[17.375]], [[9.0]], [[3.65625]], [[3.421875]], [[20.125]], [[10.4375]], [[17.125]], [[20.875]], [[13.75]], [[8.3125]], [[7.375]], [[8.5625]], [[4.8125]], [[17.625]], [[42.75]], [[1.953125]], [[2.0]], [[8.4375]], [[1.421875]], [[15.875]], [[14.125]], [[20.375]], [[0.640625]], [[20.25]], [[14.5]], [[13.5625]], [[12.625]], [[14.9375]], [[11.5]], [[8.3125]], [[28.25]], [[24.125]], [[4.25]], [[1.1953125]], [[4.5625]], [[3.96875]], [[17.0]], [[19.25]], [[6.0]], [[8.625]], [[27.5]], [[14.4375]], [[10.0625]], [[11.0]], [[7.96875]], [[1.46875]], [[21.375]], [[3.125]], [[22.875]], [[13.9375]], [[25.25]], [[6.84375]], [[4.53125]], [[6.40625]], [[3.15625]], [[10.5625]], [[14.625]], [[34.75]], [[3.1875]], [[5.0]], [[3.109375]], [[12.5625]], [[2.546875]], [[24.0]], [[5.5625]], [[27.75]], [[11.6875]], [[16.125]], [[11.875]], [[1.921875]], [[14.1875]], [[9.75]], [[25.5]], [[7.40625]], [[23.5]], [[5.53125]], [[16.5]], [[15.8125]], [[15.5625]], [[0.8125]], [[35.25]], [[32.0]], [[16.0]], [[5.71875]], [[1.203125]], [[28.375]], [[15.875]], [[17.75]], [[34.75]], [[35.5]], [[13.9375]], [[1.828125]], [[4.96875]], [[13.125]], [[8.6875]], [[8.75]], [[21.75]], [[23.125]], [[36.75]], [[10.5625]], [[3.546875]], [[3.0625]], [[20.125]], [[8.125]]]}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.39453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.62109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.56640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[11.6875]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.7.feed_forward.shared_expert.down_proj": {"inputs": [[[7.8125]]], "outputs": [[[0.5703125]], [[2.4457795122442452e+38]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.8.self_attn.qkv_proj": {"inputs": [[[15.125]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.8.self_attn.o_proj": {"inputs": [[[2.1875]]], "outputs": [[[1.9375]], [[2.109375]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.8.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.76953125]], [[17.625]]]}, "language_model.model.layers.8.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.921875]]]}, "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.76953125]]]}, "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.421875]]]}, "language_model.model.layers.8.self_attn.attn.impl.k_cache": {"inputs": [[[17.625]]]}, "language_model.model.layers.8.self_attn.attn.impl.v_cache": {"inputs": [[[2.921875]]]}, "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.25]], [[17.625]], [[2.875]]], "outputs": [[[2.1875]], [[1.0]]]}, "language_model.model.layers.8.feed_forward.gate_up_proj": {"inputs": [[[8.25]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.8.feed_forward.down_proj": {"inputs": [[[28.375]]], "outputs": [[[2.5625]], [[9.25]]], "params": {"weight": [[0.5703125]]}}, "language_model.model.layers.9.self_attn.qkv_proj": {"inputs": [[[15.75]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.9.self_attn.o_proj": {"inputs": [[[4.21875]]], "outputs": [[[1.3125]], [[2.875]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.9.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.72265625]], [[13.1875]]]}, "language_model.model.layers.9.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.03125]]]}, "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.72265625]]]}, "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.15625]]]}, "language_model.model.layers.9.self_attn.attn.impl.k_cache": {"inputs": [[[13.1875]]]}, "language_model.model.layers.9.self_attn.attn.impl.v_cache": {"inputs": [[[5.03125]]]}, "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.5625]], [[13.1875]], [[5.03125]]], "outputs": [[[4.21875]], [[1.0]]]}, "language_model.model.layers.9.feed_forward.router": {"inputs": [[[11.75]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op": {"inputs": [[[11.75]]], "outputs": [[[3.53125]], [[11.9375]], [[13.625]], [[10.9375]], [[35.0]], [[24.875]], [[39.0]], [[22.875]], [[21.5]], [[43.25]], [[21.125]], [[27.875]], [[16.25]], [[33.25]], [[27.125]], [[7.59375]], [[19.125]], [[33.25]], [[31.375]], [[37.5]], [[13.875]], [[17.0]], [[14.0625]], [[31.25]], [[18.375]], [[14.8125]], [[19.625]], [[46.75]], [[33.25]], [[15.25]], [[37.25]], [[23.0]], [[9.9375]], [[28.125]], [[36.5]], [[41.25]], [[10.8125]], [[34.0]], [[26.25]], [[7.25]], [[18.75]], [[6.96875]], [[11.6875]], [[21.25]], [[49.75]], [[10.8125]], [[12.3125]], [[20.5]], [[30.0]], [[23.875]], [[25.375]], [[12.375]], [[24.625]], [[13.6875]], [[10.9375]], [[18.75]], [[13.5625]], [[49.5]], [[22.625]], [[38.25]], [[46.75]], [[22.125]], [[15.75]], [[27.5]], [[22.75]], [[18.5]], [[16.375]], [[45.75]], [[23.75]], [[4.84375]], [[18.0]], [[25.5]], [[36.0]], [[8.75]], [[31.125]], [[4.5625]], [[23.25]], [[8.875]], [[32.75]], [[8.375]], [[9.625]], [[20.625]], [[23.125]], [[34.25]], [[19.375]], [[12.4375]], [[36.25]], [[7.15625]], [[17.625]], [[21.5]], [[36.5]], [[4.8125]], [[15.625]], [[25.125]], [[17.875]], [[23.875]], [[19.25]], [[17.0]], [[16.875]], [[9.375]], [[18.5]], [[36.25]], [[18.0]], [[33.5]], [[19.625]], [[15.5]], [[21.5]], [[33.0]], [[40.75]], [[31.125]], [[23.0]], [[42.75]], [[3.21875]], [[22.5]], [[9.25]], [[33.25]], [[38.5]], [[19.5]], [[13.0625]], [[13.9375]], [[13.5]], [[33.0]], [[6.34375]], [[20.25]], [[65.5]], [[27.625]], [[15.9375]], [[8.5625]], [[20.5]]]}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.53515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[11.75]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.9.feed_forward.shared_expert.down_proj": {"inputs": [[[10.25]]], "outputs": [[[0.796875]], [[8.706443372391199e+37]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.10.self_attn.qkv_proj": {"inputs": [[[16.125]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.10.self_attn.o_proj": {"inputs": [[[2.21875]]], "outputs": [[[1.6015625]], [[3.09375]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.10.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.83984375]], [[13.1875]]]}, "language_model.model.layers.10.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.515625]]]}, "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.83984375]]]}, "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.21875]]]}, "language_model.model.layers.10.self_attn.attn.impl.k_cache": {"inputs": [[[13.1875]]]}, "language_model.model.layers.10.self_attn.attn.impl.v_cache": {"inputs": [[[2.515625]]]}, "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.75]], [[13.1875]], [[2.515625]]], "outputs": [[[2.15625]], [[1.0]]]}, "language_model.model.layers.10.feed_forward.gate_up_proj": {"inputs": [[[9.875]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.10.feed_forward.down_proj": {"inputs": [[[10.125]]], "outputs": [[[6.90625]], [[11.3125]]], "params": {"weight": [[0.48828125]]}}, "language_model.model.layers.11.self_attn.qkv_proj": {"inputs": [[[13.5625]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.11.self_attn.o_proj": {"inputs": [[[1.390625]]], "outputs": [[[8.0625]], [[7.875]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.11.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.6484375]], [[8.1875]]]}, "language_model.model.layers.11.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.546875]]]}, "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.6484375]]]}, "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.1875]]]}, "language_model.model.layers.11.self_attn.attn.impl.k_cache": {"inputs": [[[8.1875]]]}, "language_model.model.layers.11.self_attn.attn.impl.v_cache": {"inputs": [[[3.546875]]]}, "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[7.6875]], [[8.1875]], [[3.546875]]], "outputs": [[[1.390625]], [[1.0]]]}, "language_model.model.layers.11.feed_forward.router": {"inputs": [[[13.375]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op": {"inputs": [[[13.375]]], "outputs": [[[3.46875]], [[27.375]], [[13.6875]], [[22.375]], [[14.0625]], [[15.875]], [[21.875]], [[0.97265625]], [[25.25]], [[24.75]], [[14.4375]], [[26.375]], [[19.625]], [[48.0]], [[45.75]], [[17.0]], [[48.25]], [[38.75]], [[22.625]], [[41.5]], [[14.75]], [[12.3125]], [[26.25]], [[26.25]], [[12.6875]], [[28.5]], [[47.5]], [[29.25]], [[24.125]], [[7.59375]], [[42.25]], [[25.0]], [[10.25]], [[8.75]], [[15.375]], [[23.75]], [[31.75]], [[19.375]], [[28.0]], [[16.75]], [[25.875]], [[4.875]], [[7.71875]], [[12.8125]], [[23.75]], [[13.875]], [[2.0625]], [[8.3125]], [[35.75]], [[41.25]], [[8.0625]], [[13.8125]], [[28.5]], [[4.5625]], [[26.625]], [[50.75]], [[26.0]], [[12.5]], [[7.625]], [[37.75]], [[28.0]], [[25.125]], [[14.5625]], [[28.5]], [[36.0]], [[13.9375]], [[23.625]], [[25.25]], [[38.0]], [[15.9375]], [[32.0]], [[10.3125]], [[9.1875]], [[1.6875]], [[56.5]], [[21.625]], [[34.5]], [[29.375]], [[7.4375]], [[30.125]], [[38.75]], [[21.875]], [[24.5]], [[12.4375]], [[11.4375]], [[2.1875]], [[2.65625]], [[43.75]], [[6.8125]], [[20.5]], [[22.0]], [[34.25]], [[15.375]], [[28.0]], [[57.5]], [[14.0]], [[19.375]], [[24.625]], [[19.75]], [[13.75]], [[30.25]], [[8.375]], [[45.5]], [[23.25]], [[12.5625]], [[28.625]], [[34.0]], [[23.625]], [[8.0625]], [[12.1875]], [[8.375]], [[32.75]], [[5.875]], [[2.953125]], [[36.5]], [[36.0]], [[43.75]], [[40.5]], [[47.0]], [[11.625]], [[36.25]], [[23.375]], [[13.8125]], [[14.4375]], [[18.0]], [[5.125]], [[16.75]], [[31.5]], [[19.25]]]}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[13.375]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.11.feed_forward.shared_expert.down_proj": {"inputs": [[[12.8125]]], "outputs": [[[0.76171875]], [[2.9110093107689658e+38]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.12.self_attn.qkv_proj": {"inputs": [[[18.625]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.12.self_attn.o_proj": {"inputs": [[[2.796875]]], "outputs": [[[1.3359375]], [[3.640625]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.12.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.046875]], [[18.25]]]}, "language_model.model.layers.12.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.265625]]]}, "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.046875]]]}, "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.96875]]]}, "language_model.model.layers.12.self_attn.attn.impl.k_cache": {"inputs": [[[18.25]]]}, "language_model.model.layers.12.self_attn.attn.impl.v_cache": {"inputs": [[[3.75]]]}, "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.9375]], [[18.25]], [[3.75]]], "outputs": [[[2.796875]], [[1.0]]]}, "language_model.model.layers.12.feed_forward.gate_up_proj": {"inputs": [[[10.75]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.12.feed_forward.down_proj": {"inputs": [[[13.625]]], "outputs": [[[9.0]], [[17.375]]], "params": {"weight": [[0.60546875]]}}, "language_model.model.layers.13.self_attn.qkv_proj": {"inputs": [[[20.875]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.13.self_attn.o_proj": {"inputs": [[[4.5625]]], "outputs": [[[1.6875]], [[6.9375]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.13.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.83984375]], [[17.25]]]}, "language_model.model.layers.13.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.5625]]]}, "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.83984375]]]}, "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.671875]]]}, "language_model.model.layers.13.self_attn.attn.impl.k_cache": {"inputs": [[[17.25]]]}, "language_model.model.layers.13.self_attn.attn.impl.v_cache": {"inputs": [[[4.5625]]]}, "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.1875]], [[17.25]], [[4.5625]]], "outputs": [[[4.5625]], [[1.0]]]}, "language_model.model.layers.13.feed_forward.router": {"inputs": [[[17.625]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op": {"inputs": [[[17.625]]], "outputs": [[[4.5625]], [[21.625]], [[32.5]], [[23.125]], [[36.25]], [[37.5]], [[11.9375]], [[24.75]], [[11.625]], [[11.5625]], [[4.4375]], [[36.5]], [[24.25]], [[9.8125]], [[19.375]], [[46.5]], [[30.25]], [[16.875]], [[25.875]], [[5.0625]], [[39.75]], [[22.5]], [[24.5]], [[9.75]], [[6.71875]], [[13.1875]], [[51.5]], [[17.375]], [[52.0]], [[34.0]], [[16.75]], [[7.65625]], [[18.0]], [[38.75]], [[16.75]], [[30.875]], [[22.375]], [[49.5]], [[77.5]], [[20.75]], [[4.4375]], [[45.75]], [[25.5]], [[24.875]], [[13.375]], [[35.0]], [[41.75]], [[24.625]], [[24.625]], [[16.0]], [[27.375]], [[25.25]], [[45.75]], [[30.5]], [[54.5]], [[7.09375]], [[34.0]], [[36.25]], [[30.0]], [[25.375]], [[32.0]], [[16.625]], [[20.125]], [[7.375]], [[21.625]], [[10.3125]], [[57.25]], [[7.28125]], [[31.5]], [[34.5]], [[7.84375]], [[50.0]], [[34.5]], [[44.75]], [[11.0]], [[27.5]], [[25.625]], [[57.25]], [[29.625]], [[25.25]], [[20.125]], [[9.375]], [[49.25]], [[8.875]], [[69.0]], [[50.0]], [[30.625]], [[29.625]], [[9.125]], [[3.203125]], [[35.5]], [[64.0]], [[42.25]], [[6.25]], [[34.75]], [[51.5]], [[15.625]], [[13.3125]], [[11.3125]], [[27.375]], [[39.25]], [[6.46875]], [[3.046875]], [[33.75]], [[25.75]], [[37.75]], [[42.75]], [[31.5]], [[48.75]], [[14.0]], [[26.625]], [[24.375]], [[23.0]], [[34.75]], [[55.75]], [[44.5]], [[52.75]], [[27.25]], [[36.25]], [[42.5]], [[24.5]], [[39.75]], [[28.0]], [[40.5]], [[9.6875]], [[21.5]], [[29.5]], [[5.0]], [[44.0]]]}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[17.625]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.13.feed_forward.shared_expert.down_proj": {"inputs": [[[18.125]]], "outputs": [[[1.1875]], [[2.9110093107689658e+38]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.14.self_attn.qkv_proj": {"inputs": [[[18.75]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.14.self_attn.o_proj": {"inputs": [[[3.46875]]], "outputs": [[[2.140625]], [[4.875]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.14.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.78515625]], [[13.875]]]}, "language_model.model.layers.14.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.15625]]]}, "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.78515625]]]}, "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.078125]]]}, "language_model.model.layers.14.self_attn.attn.impl.k_cache": {"inputs": [[[13.875]]]}, "language_model.model.layers.14.self_attn.attn.impl.v_cache": {"inputs": [[[4.15625]]]}, "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.375]], [[13.875]], [[4.15625]]], "outputs": [[[3.46875]], [[1.0]]]}, "language_model.model.layers.14.feed_forward.gate_up_proj": {"inputs": [[[12.0625]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.14.feed_forward.down_proj": {"inputs": [[[13.8125]]], "outputs": [[[4.65625]], [[28.25]]], "params": {"weight": [[0.51171875]]}}, "language_model.model.layers.15.self_attn.qkv_proj": {"inputs": [[[20.875]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.15.self_attn.o_proj": {"inputs": [[[3.1875]]], "outputs": [[[21.875]], [[23.0]]], "params": {"weight": [[0.84765625]]}}, "language_model.model.layers.15.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.609375]], [[10.25]]]}, "language_model.model.layers.15.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.6875]]]}, "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.609375]]]}, "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.25]]]}, "language_model.model.layers.15.self_attn.attn.impl.k_cache": {"inputs": [[[10.25]]]}, "language_model.model.layers.15.self_attn.attn.impl.v_cache": {"inputs": [[[4.6875]]]}, "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[6.9375]], [[10.25]], [[4.6875]]], "outputs": [[[3.1875]], [[1.0]]]}, "language_model.model.layers.15.feed_forward.router": {"inputs": [[[19.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op": {"inputs": [[[19.0]]], "outputs": [[[4.21875]], [[13.0625]], [[11.3125]], [[38.25]], [[16.25]], [[23.875]], [[40.75]], [[25.25]], [[39.5]], [[29.25]], [[48.25]], [[43.0]], [[19.75]], [[29.625]], [[21.875]], [[5.84375]], [[26.5]], [[35.75]], [[20.125]], [[21.125]], [[9.5625]], [[58.0]], [[31.5]], [[31.25]], [[31.25]], [[13.375]], [[5.34375]], [[19.125]], [[25.625]], [[49.75]], [[18.125]], [[33.5]], [[22.625]], [[13.1875]], [[12.1875]], [[20.625]], [[22.375]], [[9.8125]], [[30.75]], [[73.0]], [[40.0]], [[54.75]], [[47.75]], [[38.25]], [[34.0]], [[61.25]], [[30.125]], [[23.5]], [[22.0]], [[17.125]], [[39.25]], [[29.0]], [[31.375]], [[29.375]], [[18.0]], [[49.75]], [[18.375]], [[18.75]], [[53.0]], [[34.75]], [[34.25]], [[24.125]], [[56.0]], [[47.5]], [[41.5]], [[27.5]], [[68.5]], [[33.0]], [[53.25]], [[10.0]], [[33.5]], [[58.5]], [[20.5]], [[26.625]], [[65.5]], [[42.0]], [[30.0]], [[49.75]], [[43.0]], [[33.0]], [[7.25]], [[40.25]], [[31.375]], [[18.0]], [[16.75]], [[7.28125]], [[19.75]], [[22.25]], [[10.1875]], [[22.75]], [[31.75]], [[7.09375]], [[19.0]], [[25.75]], [[34.75]], [[62.5]], [[31.625]], [[28.625]], [[25.25]], [[11.0]], [[47.75]], [[58.25]], [[44.25]], [[37.25]], [[55.5]], [[50.5]], [[27.25]], [[9.4375]], [[31.5]], [[44.75]], [[28.0]], [[52.0]], [[42.25]], [[52.0]], [[16.0]], [[14.875]], [[49.5]], [[69.0]], [[14.1875]], [[25.625]], [[28.5]], [[77.0]], [[31.25]], [[16.375]], [[39.75]], [[56.75]], [[8.5]], [[16.875]], [[40.5]]]}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[19.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.15.feed_forward.shared_expert.down_proj": {"inputs": [[[20.375]]], "outputs": [[[9.3125]], [[2.9110093107689658e+38]]], "params": {"weight": [[0.76171875]]}}, "language_model.model.layers.16.self_attn.qkv_proj": {"inputs": [[[20.375]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.16.self_attn.o_proj": {"inputs": [[[2.875]]], "outputs": [[[2.75]], [[6.28125]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.16.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.9765625]], [[13.25]]]}, "language_model.model.layers.16.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.578125]]]}, "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.9765625]]]}, "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.5625]]]}, "language_model.model.layers.16.self_attn.attn.impl.k_cache": {"inputs": [[[13.25]]]}, "language_model.model.layers.16.self_attn.attn.impl.v_cache": {"inputs": [[[3.578125]]]}, "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.3125]], [[13.25]], [[3.578125]]], "outputs": [[[2.875]], [[1.0]]]}, "language_model.model.layers.16.feed_forward.gate_up_proj": {"inputs": [[[14.8125]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.16.feed_forward.down_proj": {"inputs": [[[15.0625]]], "outputs": [[[7.03125]], [[22.625]]], "params": {"weight": [[0.423828125]]}}, "language_model.model.layers.17.self_attn.qkv_proj": {"inputs": [[[19.875]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.17.self_attn.o_proj": {"inputs": [[[4.03125]]], "outputs": [[[17.5]], [[20.75]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.17.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.98828125]], [[14.6875]]]}, "language_model.model.layers.17.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.78125]]]}, "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.98828125]]]}, "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.578125]]]}, "language_model.model.layers.17.self_attn.attn.impl.k_cache": {"inputs": [[[14.6875]]]}, "language_model.model.layers.17.self_attn.attn.impl.v_cache": {"inputs": [[[4.78125]]]}, "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.1875]], [[14.6875]], [[4.78125]]], "outputs": [[[4.03125]], [[1.0]]]}, "language_model.model.layers.17.feed_forward.router": {"inputs": [[[23.875]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op": {"inputs": [[[23.875]]], "outputs": [[[5.625]], [[27.0]], [[17.375]], [[29.875]], [[25.375]], [[61.25]], [[29.625]], [[54.25]], [[37.75]], [[71.5]], [[6.78125]], [[6.71875]], [[18.5]], [[20.375]], [[22.125]], [[24.75]], [[45.25]], [[44.25]], [[31.875]], [[34.75]], [[20.0]], [[38.75]], [[41.75]], [[31.625]], [[76.0]], [[9.375]], [[49.75]], [[16.625]], [[29.125]], [[48.0]], [[42.25]], [[25.625]], [[11.875]], [[7.8125]], [[31.125]], [[31.5]], [[52.0]], [[22.75]], [[18.0]], [[11.8125]], [[52.75]], [[19.125]], [[44.75]], [[8.375]], [[22.625]], [[36.0]], [[44.25]], [[42.75]], [[63.5]], [[34.75]], [[30.375]], [[49.75]], [[62.0]], [[21.5]], [[15.0]], [[37.0]], [[4.4375]], [[11.375]], [[34.0]], [[17.625]], [[15.0625]], [[15.75]], [[29.5]], [[25.375]], [[15.0625]], [[44.75]], [[30.375]], [[15.0]], [[61.75]], [[26.75]], [[11.5]], [[30.875]], [[32.5]], [[67.5]], [[27.0]], [[47.75]], [[28.375]], [[44.0]], [[75.5]], [[11.1875]], [[45.25]], [[17.375]], [[55.25]], [[29.75]], [[46.25]], [[36.0]], [[40.25]], [[24.5]], [[24.375]], [[16.375]], [[50.25]], [[33.5]], [[38.0]], [[18.625]], [[57.25]], [[9.1875]], [[43.5]], [[10.0625]], [[3.5]], [[82.5]], [[24.5]], [[38.0]], [[25.25]], [[28.75]], [[53.25]], [[48.25]], [[11.625]], [[18.25]], [[62.5]], [[15.5625]], [[19.5]], [[53.75]], [[52.0]], [[37.5]], [[64.0]], [[25.25]], [[8.3125]], [[57.25]], [[30.0]], [[22.625]], [[55.0]], [[42.75]], [[28.0]], [[22.125]], [[10.75]], [[25.25]], [[7.375]], [[16.625]], [[57.25]]]}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.384765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0751953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.875]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.17.feed_forward.shared_expert.down_proj": {"inputs": [[[20.5]]], "outputs": [[[5.34375]], [[2.9110093107689658e+38]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.18.self_attn.qkv_proj": {"inputs": [[[22.125]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.18.self_attn.o_proj": {"inputs": [[[5.75]]], "outputs": [[[6.375]], [[11.5625]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.18.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.93359375]], [[17.5]]]}, "language_model.model.layers.18.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.46875]]]}, "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.93359375]]]}, "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.15625]]]}, "language_model.model.layers.18.self_attn.attn.impl.k_cache": {"inputs": [[[17.5]]]}, "language_model.model.layers.18.self_attn.attn.impl.v_cache": {"inputs": [[[6.46875]]]}, "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.8125]], [[17.5]], [[6.46875]]], "outputs": [[[5.75]], [[1.0]]]}, "language_model.model.layers.18.feed_forward.gate_up_proj": {"inputs": [[[18.5]]], "params": {"weight": [[0.59375]]}}, "language_model.model.layers.18.feed_forward.down_proj": {"inputs": [[[19.125]]], "outputs": [[[8.25]], [[358.0]]], "params": {"weight": [[0.64453125]]}}, "language_model.model.layers.19.self_attn.qkv_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.19.self_attn.o_proj": {"inputs": [[[5.3125]]], "outputs": [[[31.625]], [[36.75]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.19.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8046875]], [[15.0]]]}, "language_model.model.layers.19.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.71875]]]}, "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8046875]]]}, "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.3125]]]}, "language_model.model.layers.19.self_attn.attn.impl.k_cache": {"inputs": [[[15.0]]]}, "language_model.model.layers.19.self_attn.attn.impl.v_cache": {"inputs": [[[6.71875]]]}, "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.125]], [[15.0]], [[6.71875]]], "outputs": [[[5.3125]], [[1.0]]]}, "language_model.model.layers.19.feed_forward.router": {"inputs": [[[23.25]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op": {"inputs": [[[23.25]]], "outputs": [[[5.125]], [[22.125]], [[43.25]], [[15.8125]], [[40.25]], [[32.75]], [[14.9375]], [[18.125]], [[52.5]], [[12.5]], [[17.875]], [[9.0]], [[28.0]], [[22.875]], [[23.0]], [[34.5]], [[7.0]], [[7.1875]], [[42.5]], [[61.5]], [[59.0]], [[9.375]], [[34.5]], [[30.625]], [[15.25]], [[34.5]], [[31.25]], [[23.375]], [[42.5]], [[36.25]], [[31.25]], [[31.625]], [[34.75]], [[35.5]], [[16.125]], [[42.75]], [[40.0]], [[14.4375]], [[27.0]], [[36.25]], [[21.75]], [[32.5]], [[14.125]], [[33.5]], [[40.75]], [[54.0]], [[103.0]], [[16.25]], [[13.3125]], [[30.625]], [[40.5]], [[9.6875]], [[52.5]], [[5.5]], [[38.5]], [[57.25]], [[26.625]], [[25.875]], [[34.25]], [[37.5]], [[29.125]], [[44.5]], [[50.0]], [[7.59375]], [[24.75]], [[53.0]], [[66.0]], [[49.25]], [[22.625]], [[30.0]], [[9.9375]], [[57.5]], [[30.375]], [[26.75]], [[40.25]], [[30.875]], [[20.625]], [[17.75]], [[29.75]], [[32.5]], [[8.625]], [[40.75]], [[16.875]], [[30.625]], [[49.0]], [[68.5]], [[28.75]], [[16.875]], [[10.875]], [[18.125]], [[31.25]], [[22.0]], [[49.5]], [[48.25]], [[39.25]], [[25.125]], [[26.375]], [[20.875]], [[36.5]], [[15.75]], [[13.4375]], [[65.5]], [[29.625]], [[29.875]], [[51.25]], [[29.875]], [[23.125]], [[15.8125]], [[43.5]], [[24.375]], [[24.375]], [[20.25]], [[21.875]], [[39.5]], [[38.25]], [[15.625]], [[10.6875]], [[21.5]], [[34.25]], [[34.25]], [[61.75]], [[11.875]], [[9.0625]], [[18.0]], [[28.625]], [[11.5]], [[10.25]], [[28.375]], [[34.0]]]}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.439453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.419921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.490234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.4765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.25]]], "params": {"weight": [[0.423828125]]}}, "language_model.model.layers.19.feed_forward.shared_expert.down_proj": {"inputs": [[[22.125]]], "outputs": [[[13.125]], [[2.9110093107689658e+38]]], "params": {"weight": [[0.77734375]]}}, "language_model.model.layers.20.self_attn.qkv_proj": {"inputs": [[[52.5]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.20.self_attn.o_proj": {"inputs": [[[10.25]]], "outputs": [[[9.5]], [[18.375]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.20.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.9375]], [[16.25]]]}, "language_model.model.layers.20.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.125]]]}, "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.9375]]]}, "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.75]]]}, "language_model.model.layers.20.self_attn.attn.impl.k_cache": {"inputs": [[[16.25]]]}, "language_model.model.layers.20.self_attn.attn.impl.v_cache": {"inputs": [[[11.125]]]}, "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.25]], [[16.25]], [[11.125]]], "outputs": [[[10.25]], [[1.0]]]}, "language_model.model.layers.20.feed_forward.gate_up_proj": {"inputs": [[[41.5]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.20.feed_forward.down_proj": {"inputs": [[[17.5]]], "outputs": [[[9.4375]], [[38.5]]], "params": {"weight": [[0.95703125]]}}, "language_model.model.layers.21.self_attn.qkv_proj": {"inputs": [[[56.25]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.21.self_attn.o_proj": {"inputs": [[[4.21875]]], "outputs": [[[25.125]], [[37.75]]], "params": {"weight": [[0.388671875]]}}, "language_model.model.layers.21.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1015625]], [[16.125]]]}, "language_model.model.layers.21.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.375]]]}, "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1015625]]]}, "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.625]]]}, "language_model.model.layers.21.self_attn.attn.impl.k_cache": {"inputs": [[[16.125]]]}, "language_model.model.layers.21.self_attn.attn.impl.v_cache": {"inputs": [[[8.375]]]}, "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.8125]], [[16.125]], [[8.375]]], "outputs": [[[4.21875]], [[1.0]]]}, "language_model.model.layers.21.feed_forward.router": {"inputs": [[[24.5]]], "params": {"weight": [[0.45703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op": {"inputs": [[[24.5]]], "outputs": [[[6.09375]], [[29.25]], [[95.0]], [[40.25]], [[46.25]], [[9.75]], [[42.75]], [[18.25]], [[24.375]], [[21.875]], [[29.125]], [[11.0625]], [[9.5625]], [[18.125]], [[78.5]], [[20.75]], [[73.5]], [[55.75]], [[34.75]], [[23.375]], [[56.0]], [[43.25]], [[47.25]], [[22.375]], [[23.375]], [[32.5]], [[6.4375]], [[8.875]], [[11.375]], [[18.375]], [[21.0]], [[8.3125]], [[46.25]], [[45.75]], [[21.125]], [[11.5]], [[12.625]], [[18.5]], [[19.75]], [[32.5]], [[19.875]], [[7.0625]], [[26.375]], [[20.875]], [[7.0625]], [[27.125]], [[26.75]], [[17.125]], [[31.125]], [[34.25]], [[26.625]], [[37.25]], [[38.0]], [[34.5]], [[44.0]], [[5.25]], [[60.75]], [[43.0]], [[16.0]], [[10.875]], [[59.0]], [[45.25]], [[35.0]], [[34.0]], [[53.5]], [[14.0]], [[54.75]], [[16.0]], [[25.0]], [[25.125]], [[9.1875]], [[27.875]], [[14.3125]], [[15.0]], [[30.625]], [[18.5]], [[12.0]], [[52.75]], [[16.625]], [[24.25]], [[17.25]], [[26.5]], [[21.0]], [[50.25]], [[36.25]], [[28.25]], [[55.25]], [[18.125]], [[15.4375]], [[7.46875]], [[23.75]], [[20.375]], [[11.625]], [[47.0]], [[13.9375]], [[10.1875]], [[26.25]], [[37.25]], [[42.25]], [[16.0]], [[23.125]], [[22.0]], [[54.5]], [[41.5]], [[47.0]], [[28.375]], [[29.125]], [[26.875]], [[47.5]], [[36.0]], [[23.0]], [[45.0]], [[37.0]], [[28.75]], [[25.5]], [[24.25]], [[13.1875]], [[31.5]], [[21.0]], [[22.25]], [[32.0]], [[8.625]], [[52.5]], [[25.25]], [[20.75]], [[28.5]], [[45.0]], [[32.75]], [[59.75]]]}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.34765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[24.5]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.21.feed_forward.shared_expert.down_proj": {"inputs": [[[42.5]]], "outputs": [[[42.75]], [[2.9110093107689658e+38]]], "params": {"weight": [[0.9609375]]}}, "language_model.model.layers.22.self_attn.qkv_proj": {"inputs": [[[55.75]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.22.self_attn.o_proj": {"inputs": [[[6.3125]]], "outputs": [[[17.25]], [[23.25]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.22.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.9375]], [[17.125]]]}, "language_model.model.layers.22.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.03125]]]}, "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.9375]]]}, "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.25]]]}, "language_model.model.layers.22.self_attn.attn.impl.k_cache": {"inputs": [[[17.125]]]}, "language_model.model.layers.22.self_attn.attn.impl.v_cache": {"inputs": [[[7.03125]]]}, "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.6875]], [[17.125]], [[7.03125]]], "outputs": [[[6.3125]], [[1.0]]]}, "language_model.model.layers.22.feed_forward.gate_up_proj": {"inputs": [[[44.75]]], "params": {"weight": [[0.421875]]}}, "language_model.model.layers.22.feed_forward.down_proj": {"inputs": [[[22.75]]], "outputs": [[[12.0]], [[80.5]]], "params": {"weight": [[0.6015625]]}}, "language_model.model.layers.23.self_attn.qkv_proj": {"inputs": [[[67.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.23.self_attn.o_proj": {"inputs": [[[5.5]]], "outputs": [[[50.5]], [[61.5]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.23.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8828125]], [[14.0]]]}, "language_model.model.layers.23.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.625]]]}, "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8828125]]]}, "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.375]]]}, "language_model.model.layers.23.self_attn.attn.impl.k_cache": {"inputs": [[[14.0]]]}, "language_model.model.layers.23.self_attn.attn.impl.v_cache": {"inputs": [[[12.625]]]}, "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.0625]], [[14.0]], [[12.625]]], "outputs": [[[5.5]], [[1.0]]]}, "language_model.model.layers.23.feed_forward.router": {"inputs": [[[23.625]]], "params": {"weight": [[0.474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op": {"inputs": [[[23.625]]], "outputs": [[[207.0]], [[70.0]], [[6.6875]], [[37.0]], [[55.25]], [[6.96875]], [[34.75]], [[23.75]], [[43.0]], [[69.5]], [[21.25]], [[25.125]], [[64.5]], [[41.5]], [[13.875]], [[20.875]], [[48.5]], [[36.75]], [[18.75]], [[39.25]], [[36.5]], [[54.5]], [[58.25]], [[23.25]], [[32.25]], [[50.5]], [[62.25]], [[34.0]], [[76.5]], [[7.59375]], [[13.125]], [[24.5]], [[26.375]], [[14.0625]], [[11.1875]], [[26.25]], [[41.0]], [[24.75]], [[17.125]], [[12.125]], [[63.0]], [[61.75]], [[11.5]], [[63.5]], [[10.1875]], [[20.25]], [[6.8125]], [[48.0]], [[42.25]], [[8.75]], [[71.0]], [[19.375]], [[39.5]], [[30.375]], [[19.875]], [[15.0]], [[36.75]], [[244.0]], [[9.9375]], [[51.25]], [[25.125]], [[29.25]], [[36.5]], [[9.25]], [[15.875]], [[20.75]], [[12.6875]], [[43.75]], [[57.0]], [[31.125]], [[43.25]], [[7.53125]], [[21.0]], [[47.25]], [[61.5]], [[11.0]], [[53.25]], [[52.75]], [[37.75]], [[47.0]], [[48.25]], [[31.125]], [[16.25]], [[43.25]], [[23.0]], [[6.5]], [[47.0]], [[31.5]], [[8.875]], [[27.125]], [[15.0]], [[32.75]], [[46.0]], [[23.125]], [[15.125]], [[23.75]], [[41.5]], [[25.0]], [[27.625]], [[21.875]], [[36.25]], [[8.5]], [[13.8125]], [[14.0625]], [[11.5]], [[27.875]], [[45.75]], [[37.25]], [[31.75]], [[7.15625]], [[89.0]], [[12.375]], [[93.0]], [[7.4375]], [[30.0]], [[25.125]], [[25.25]], [[15.375]], [[86.5]], [[14.0625]], [[15.0]], [[13.625]], [[43.25]], [[41.5]], [[53.25]], [[17.5]], [[24.625]], [[12.6875]], [[8.5625]]]}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.4296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.6953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.392578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.625]]], "params": {"weight": [[0.37109375]]}}, "language_model.model.layers.23.feed_forward.shared_expert.down_proj": {"inputs": [[[31.625]]], "outputs": [[[28.5]], [[2.9110093107689658e+38]]], "params": {"weight": [[0.8671875]]}}, "language_model.model.layers.24.self_attn.qkv_proj": {"inputs": [[[60.5]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.24.self_attn.o_proj": {"inputs": [[[6.90625]]], "outputs": [[[36.0]], [[42.5]]], "params": {"weight": [[0.640625]]}}, "language_model.model.layers.24.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.15625]], [[19.25]]]}, "language_model.model.layers.24.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.5]]]}, "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.15625]]]}, "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.0625]]]}, "language_model.model.layers.24.self_attn.attn.impl.k_cache": {"inputs": [[[19.25]]]}, "language_model.model.layers.24.self_attn.attn.impl.v_cache": {"inputs": [[[7.5]]]}, "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.125]], [[19.25]], [[7.5]]], "outputs": [[[6.90625]], [[1.0]]]}, "language_model.model.layers.24.feed_forward.gate_up_proj": {"inputs": [[[40.5]]], "params": {"weight": [[0.451171875]]}}, "language_model.model.layers.24.feed_forward.down_proj": {"inputs": [[[16.75]]], "outputs": [[[13.3125]], [[62.5]]], "params": {"weight": [[0.59765625]]}}, "language_model.model.layers.25.self_attn.qkv_proj": {"inputs": [[[49.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.25.self_attn.o_proj": {"inputs": [[[4.625]]], "outputs": [[[47.75]], [[57.0]]], "params": {"weight": [[0.482421875]]}}, "language_model.model.layers.25.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.03125]], [[15.8125]]]}, "language_model.model.layers.25.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.75]]]}, "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.03125]]]}, "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.375]]]}, "language_model.model.layers.25.self_attn.attn.impl.k_cache": {"inputs": [[[16.5]]]}, "language_model.model.layers.25.self_attn.attn.impl.v_cache": {"inputs": [[[6.75]]]}, "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.25]], [[16.5]], [[6.75]]], "outputs": [[[4.625]], [[1.0]]]}, "language_model.model.layers.25.feed_forward.router": {"inputs": [[[30.375]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op": {"inputs": [[[30.375]]], "outputs": [[[5.9375]], [[53.25]], [[32.75]], [[22.75]], [[9.9375]], [[27.25]], [[44.25]], [[17.25]], [[26.0]], [[19.5]], [[12.125]], [[5.34375]], [[23.75]], [[35.5]], [[59.0]], [[45.25]], [[43.5]], [[24.75]], [[18.875]], [[40.75]], [[41.75]], [[40.25]], [[44.0]], [[51.75]], [[26.0]], [[8.75]], [[14.0]], [[46.25]], [[13.5625]], [[12.0]], [[23.5]], [[20.625]], [[13.4375]], [[8.1875]], [[82.0]], [[21.875]], [[18.125]], [[19.625]], [[29.25]], [[10.625]], [[53.5]], [[18.375]], [[41.5]], [[40.5]], [[42.75]], [[22.125]], [[47.75]], [[76.0]], [[10.3125]], [[13.0]], [[53.5]], [[23.375]], [[20.375]], [[9.3125]], [[27.125]], [[31.375]], [[25.25]], [[40.0]], [[16.0]], [[11.375]], [[10.375]], [[22.5]], [[61.0]], [[19.5]], [[24.75]], [[85.0]], [[11.8125]], [[40.0]], [[26.625]], [[18.125]], [[13.3125]], [[9.75]], [[14.5]], [[33.25]], [[26.375]], [[77.0]], [[44.0]], [[18.0]], [[17.5]], [[29.25]], [[60.5]], [[12.9375]], [[28.5]], [[36.75]], [[58.5]], [[8.875]], [[21.625]], [[43.5]], [[41.0]], [[42.0]], [[39.75]], [[24.75]], [[15.875]], [[41.75]], [[23.625]], [[10.1875]], [[28.75]], [[37.5]], [[38.75]], [[13.4375]], [[40.0]], [[18.5]], [[26.375]], [[26.25]], [[16.25]], [[25.0]], [[23.125]], [[40.75]], [[5.53125]], [[57.5]], [[35.25]], [[66.0]], [[20.375]], [[64.0]], [[46.5]], [[40.0]], [[40.0]], [[75.5]], [[35.0]], [[63.0]], [[35.5]], [[25.75]], [[44.25]], [[20.5]], [[21.375]], [[37.5]], [[17.125]], [[21.125]], [[13.3125]]]}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.32421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.4296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[30.375]]], "params": {"weight": [[0.484375]]}}, "language_model.model.layers.25.feed_forward.shared_expert.down_proj": {"inputs": [[[29.625]]], "outputs": [[[9.1875]], [[2.9110093107689658e+38]]], "params": {"weight": [[0.60546875]]}}, "language_model.model.layers.26.self_attn.qkv_proj": {"inputs": [[[63.25]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.26.self_attn.o_proj": {"inputs": [[[4.96875]]], "outputs": [[[29.0]], [[53.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.26.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0703125]], [[16.375]]]}, "language_model.model.layers.26.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.46875]]]}, "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0703125]]]}, "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.03125]]]}, "language_model.model.layers.26.self_attn.attn.impl.k_cache": {"inputs": [[[16.375]]]}, "language_model.model.layers.26.self_attn.attn.impl.v_cache": {"inputs": [[[5.46875]]]}, "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.25]], [[16.375]], [[5.46875]]], "outputs": [[[4.96875]], [[1.0]]]}, "language_model.model.layers.26.feed_forward.gate_up_proj": {"inputs": [[[45.5]]], "params": {"weight": [[0.5703125]]}}, "language_model.model.layers.26.feed_forward.down_proj": {"inputs": [[[28.875]]], "outputs": [[[15.25]], [[103.0]]], "params": {"weight": [[0.5234375]]}}, "language_model.model.layers.27.self_attn.qkv_proj": {"inputs": [[[72.5]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.27.self_attn.o_proj": {"inputs": [[[5.59375]]], "outputs": [[[60.0]], [[66.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.27.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.96484375]], [[15.75]]]}, "language_model.model.layers.27.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.9375]]]}, "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.96484375]]]}, "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.65625]]]}, "language_model.model.layers.27.self_attn.attn.impl.k_cache": {"inputs": [[[15.75]]]}, "language_model.model.layers.27.self_attn.attn.impl.v_cache": {"inputs": [[[9.9375]]]}, "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.4375]], [[15.75]], [[9.9375]]], "outputs": [[[5.59375]], [[1.0]]]}, "language_model.model.layers.27.feed_forward.router": {"inputs": [[[30.875]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op": {"inputs": [[[30.875]]], "outputs": [[[5.5625]], [[17.75]], [[38.25]], [[41.5]], [[10.9375]], [[10.0625]], [[39.5]], [[36.25]], [[28.125]], [[67.0]], [[39.75]], [[34.25]], [[25.125]], [[45.0]], [[22.875]], [[85.0]], [[24.125]], [[28.875]], [[60.5]], [[20.25]], [[14.4375]], [[25.25]], [[12.6875]], [[21.375]], [[15.0]], [[22.125]], [[9.8125]], [[35.25]], [[30.25]], [[16.625]], [[79.5]], [[77.5]], [[17.0]], [[58.0]], [[18.875]], [[73.5]], [[17.75]], [[25.875]], [[49.5]], [[46.0]], [[28.25]], [[37.5]], [[44.0]], [[57.75]], [[27.875]], [[13.625]], [[25.75]], [[17.5]], [[20.25]], [[45.75]], [[15.25]], [[20.0]], [[10.25]], [[53.25]], [[52.5]], [[40.0]], [[35.75]], [[43.0]], [[25.375]], [[46.5]], [[9.8125]], [[18.0]], [[79.5]], [[41.75]], [[42.0]], [[56.75]], [[97.5]], [[14.8125]], [[15.875]], [[22.875]], [[13.1875]], [[21.375]], [[29.125]], [[112.5]], [[62.0]], [[30.5]], [[57.0]], [[32.75]], [[51.0]], [[70.0]], [[45.5]], [[69.5]], [[22.625]], [[27.5]], [[55.75]], [[30.75]], [[42.25]], [[9.75]], [[21.625]], [[11.125]], [[22.75]], [[13.375]], [[67.0]], [[22.0]], [[41.75]], [[25.5]], [[33.25]], [[44.25]], [[80.5]], [[25.375]], [[16.5]], [[34.25]], [[32.5]], [[15.5]], [[35.5]], [[15.5]], [[13.125]], [[12.125]], [[36.0]], [[11.375]], [[39.5]], [[21.375]], [[47.75]], [[10.625]], [[15.125]], [[21.75]], [[19.125]], [[37.25]], [[60.75]], [[28.625]], [[77.5]], [[14.75]], [[50.25]], [[29.125]], [[89.0]], [[15.0]], [[17.75]], [[11.5]], [[33.5]]]}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.48828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[30.875]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.27.feed_forward.shared_expert.down_proj": {"inputs": [[[125.0]]], "outputs": [[[82.5]], [[3.176854909925949e+38]]], "params": {"weight": [[0.66015625]]}}, "language_model.model.layers.28.self_attn.qkv_proj": {"inputs": [[[77.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.28.self_attn.o_proj": {"inputs": [[[7.3125]]], "outputs": [[[72.0]], [[74.0]]], "params": {"weight": [[0.72265625]]}}, "language_model.model.layers.28.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.34375]], [[19.5]]]}, "language_model.model.layers.28.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.53125]]]}, "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.34375]]]}, "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.3125]]]}, "language_model.model.layers.28.self_attn.attn.impl.k_cache": {"inputs": [[[19.5]]]}, "language_model.model.layers.28.self_attn.attn.impl.v_cache": {"inputs": [[[7.53125]]]}, "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.9375]], [[19.5]], [[7.53125]]], "outputs": [[[7.09375]], [[1.0]]]}, "language_model.model.layers.28.feed_forward.gate_up_proj": {"inputs": [[[40.25]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.28.feed_forward.down_proj": {"inputs": [[[22.875]]], "outputs": [[[95.0]], [[103.5]]], "params": {"weight": [[0.453125]]}}, "language_model.model.layers.29.self_attn.qkv_proj": {"inputs": [[[66.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.29.self_attn.o_proj": {"inputs": [[[5.65625]]], "outputs": [[[70.5]], [[76.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.29.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.296875]], [[17.25]]]}, "language_model.model.layers.29.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.46875]]]}, "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.296875]]]}, "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.96875]]]}, "language_model.model.layers.29.self_attn.attn.impl.k_cache": {"inputs": [[[17.25]]]}, "language_model.model.layers.29.self_attn.attn.impl.v_cache": {"inputs": [[[7.46875]]]}, "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.625]], [[17.25]], [[7.46875]]], "outputs": [[[5.65625]], [[1.0]]]}, "language_model.model.layers.29.feed_forward.router": {"inputs": [[[31.75]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op": {"inputs": [[[31.75]]], "outputs": [[[7.8125]], [[27.25]], [[52.0]], [[35.5]], [[25.625]], [[24.5]], [[38.75]], [[43.75]], [[59.25]], [[30.25]], [[29.5]], [[13.5]], [[15.75]], [[24.625]], [[23.5]], [[33.25]], [[14.75]], [[57.75]], [[56.5]], [[24.125]], [[10.5625]], [[29.0]], [[11.6875]], [[26.5]], [[19.75]], [[58.75]], [[31.125]], [[45.0]], [[17.875]], [[19.5]], [[47.25]], [[46.75]], [[10.25]], [[55.25]], [[51.0]], [[25.5]], [[43.25]], [[36.75]], [[25.5]], [[130.0]], [[51.25]], [[11.6875]], [[31.125]], [[12.5]], [[79.0]], [[34.5]], [[38.0]], [[46.75]], [[19.625]], [[46.25]], [[16.25]], [[35.25]], [[27.0]], [[94.0]], [[59.25]], [[49.75]], [[53.75]], [[39.25]], [[30.625]], [[23.375]], [[13.5]], [[28.125]], [[14.5625]], [[29.625]], [[18.375]], [[47.5]], [[56.0]], [[47.75]], [[36.25]], [[47.5]], [[16.625]], [[18.875]], [[51.5]], [[28.5]], [[35.5]], [[36.0]], [[64.5]], [[19.625]], [[49.25]], [[41.0]], [[40.25]], [[24.5]], [[20.125]], [[12.9375]], [[44.0]], [[16.125]], [[28.375]], [[21.125]], [[15.875]], [[27.5]], [[39.0]], [[23.5]], [[39.75]], [[22.0]], [[12.5625]], [[5.375]], [[24.375]], [[20.875]], [[23.5]], [[33.75]], [[12.125]], [[49.75]], [[23.625]], [[30.25]], [[61.75]], [[14.3125]], [[87.5]], [[60.5]], [[75.0]], [[16.125]], [[56.5]], [[25.0]], [[36.0]], [[79.0]], [[16.625]], [[42.0]], [[41.0]], [[22.125]], [[41.75]], [[54.0]], [[20.75]], [[27.875]], [[21.25]], [[39.75]], [[69.5]], [[13.625]], [[37.0]], [[95.5]], [[23.0]]]}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.37109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[31.75]]], "params": {"weight": [[0.4765625]]}}, "language_model.model.layers.29.feed_forward.shared_expert.down_proj": {"inputs": [[[36.75]]], "outputs": [[[8.875]], [[2.9110093107689658e+38]]], "params": {"weight": [[0.85546875]]}}, "language_model.model.layers.30.self_attn.qkv_proj": {"inputs": [[[71.5]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.30.self_attn.o_proj": {"inputs": [[[7.125]]], "outputs": [[[52.25]], [[57.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.30.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.109375]], [[16.75]]]}, "language_model.model.layers.30.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.8125]]]}, "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.109375]]]}, "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.875]]]}, "language_model.model.layers.30.self_attn.attn.impl.k_cache": {"inputs": [[[16.75]]]}, "language_model.model.layers.30.self_attn.attn.impl.v_cache": {"inputs": [[[7.8125]]]}, "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.6875]], [[16.75]], [[7.8125]]], "outputs": [[[7.125]], [[1.0]]]}, "language_model.model.layers.30.feed_forward.gate_up_proj": {"inputs": [[[32.0]]], "params": {"weight": [[0.4453125]]}}, "language_model.model.layers.30.feed_forward.down_proj": {"inputs": [[[31.75]]], "outputs": [[[18.625]], [[139.0]]], "params": {"weight": [[0.58203125]]}}, "language_model.model.layers.31.self_attn.qkv_proj": {"inputs": [[[71.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.31.self_attn.o_proj": {"inputs": [[[5.1875]]], "outputs": [[[66.5]], [[88.0]]], "params": {"weight": [[0.8046875]]}}, "language_model.model.layers.31.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.85546875]], [[11.4375]]]}, "language_model.model.layers.31.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.0625]]]}, "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.85546875]]]}, "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.125]]]}, "language_model.model.layers.31.self_attn.attn.impl.k_cache": {"inputs": [[[11.4375]]]}, "language_model.model.layers.31.self_attn.attn.impl.v_cache": {"inputs": [[[7.0625]]]}, "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.625]], [[11.4375]], [[7.0625]]], "outputs": [[[5.1875]], [[1.0]]]}, "language_model.model.layers.31.feed_forward.router": {"inputs": [[[33.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op": {"inputs": [[[33.0]]], "outputs": [[[10.0]], [[14.75]], [[93.0]], [[16.5]], [[27.375]], [[67.5]], [[75.0]], [[47.5]], [[13.75]], [[38.5]], [[38.75]], [[28.0]], [[16.375]], [[21.5]], [[11.75]], [[19.75]], [[13.125]], [[23.25]], [[32.25]], [[109.5]], [[15.875]], [[10.8125]], [[18.5]], [[35.5]], [[54.75]], [[83.0]], [[49.5]], [[58.25]], [[53.0]], [[35.75]], [[16.375]], [[23.25]], [[22.25]], [[76.5]], [[46.5]], [[19.25]], [[16.375]], [[29.875]], [[26.625]], [[28.375]], [[56.25]], [[70.0]], [[56.0]], [[31.875]], [[50.25]], [[36.75]], [[45.25]], [[48.0]], [[59.75]], [[34.75]], [[41.75]], [[44.5]], [[28.125]], [[70.0]], [[31.625]], [[53.75]], [[35.75]], [[12.125]], [[96.0]], [[32.75]], [[37.0]], [[28.375]], [[32.5]], [[22.25]], [[61.5]], [[57.5]], [[23.875]], [[27.375]], [[36.75]], [[39.25]], [[26.5]], [[42.0]], [[38.25]], [[30.375]], [[35.0]], [[12.9375]], [[13.0625]], [[30.5]], [[38.5]], [[30.75]], [[41.5]], [[41.75]], [[95.0]], [[20.5]], [[69.5]], [[17.625]], [[33.75]], [[67.5]], [[19.5]], [[53.5]], [[59.75]], [[60.0]], [[58.0]], [[39.75]], [[14.6875]], [[44.5]], [[35.5]], [[49.25]], [[18.375]], [[29.25]], [[23.0]], [[19.25]], [[15.125]], [[51.25]], [[29.375]], [[19.875]], [[7.03125]], [[31.625]], [[14.875]], [[44.5]], [[46.5]], [[49.5]], [[55.0]], [[70.0]], [[40.0]], [[33.75]], [[50.5]], [[68.5]], [[14.8125]], [[20.5]], [[88.0]], [[31.125]], [[24.625]], [[9.3125]], [[13.0]], [[65.0]], [[12.375]], [[20.625]], [[65.5]]]}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.46484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.380859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.40234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[33.0]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.31.feed_forward.shared_expert.down_proj": {"inputs": [[[59.75]]], "outputs": [[[9.0]], [[2.9110093107689658e+38]]], "params": {"weight": [[0.66796875]]}}, "language_model.model.layers.32.self_attn.qkv_proj": {"inputs": [[[53.25]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.32.self_attn.o_proj": {"inputs": [[[6.34375]]], "outputs": [[[96.0]], [[105.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.32.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.140625]], [[17.875]]]}, "language_model.model.layers.32.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.03125]]]}, "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.140625]]]}, "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.84375]]]}, "language_model.model.layers.32.self_attn.attn.impl.k_cache": {"inputs": [[[17.875]]]}, "language_model.model.layers.32.self_attn.attn.impl.v_cache": {"inputs": [[[7.03125]]]}, "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.125]], [[17.875]], [[7.03125]]], "outputs": [[[6.34375]], [[1.0]]]}, "language_model.model.layers.32.feed_forward.gate_up_proj": {"inputs": [[[32.75]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.32.feed_forward.down_proj": {"inputs": [[[52.0]]], "outputs": [[[21.125]], [[79.5]]], "params": {"weight": [[0.625]]}}, "language_model.model.layers.33.self_attn.qkv_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.33.self_attn.o_proj": {"inputs": [[[6.34375]]], "outputs": [[[97.0]], [[118.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.33.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.203125]], [[20.25]]]}, "language_model.model.layers.33.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.125]]]}, "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.203125]]]}, "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.25]]]}, "language_model.model.layers.33.self_attn.attn.impl.k_cache": {"inputs": [[[20.25]]]}, "language_model.model.layers.33.self_attn.attn.impl.v_cache": {"inputs": [[[8.125]]]}, "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.1875]], [[20.25]], [[8.125]]], "outputs": [[[6.34375]], [[1.0]]]}, "language_model.model.layers.33.feed_forward.router": {"inputs": [[[64.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op": {"inputs": [[[64.0]]], "outputs": [[[10.375]], [[54.25]], [[63.5]], [[35.5]], [[31.25]], [[26.5]], [[88.5]], [[19.75]], [[83.5]], [[39.0]], [[66.0]], [[31.375]], [[46.25]], [[93.0]], [[28.0]], [[53.5]], [[49.75]], [[35.5]], [[33.5]], [[24.375]], [[15.3125]], [[16.125]], [[16.75]], [[28.625]], [[35.75]], [[29.25]], [[29.875]], [[90.5]], [[62.25]], [[174.0]], [[54.75]], [[37.75]], [[52.0]], [[19.25]], [[55.5]], [[43.75]], [[30.875]], [[60.75]], [[15.625]], [[67.5]], [[46.75]], [[17.875]], [[32.25]], [[68.0]], [[96.0]], [[32.75]], [[19.0]], [[73.5]], [[44.25]], [[46.25]], [[60.5]], [[27.25]], [[10.4375]], [[24.75]], [[17.125]], [[123.5]], [[74.5]], [[63.0]], [[18.5]], [[43.5]], [[66.5]], [[51.0]], [[35.5]], [[31.5]], [[83.5]], [[20.0]], [[61.75]], [[33.75]], [[20.25]], [[76.0]], [[34.75]], [[114.0]], [[66.0]], [[66.0]], [[81.0]], [[27.625]], [[49.0]], [[35.0]], [[22.125]], [[9.75]], [[27.75]], [[19.125]], [[12.25]], [[47.5]], [[32.0]], [[19.875]], [[129.0]], [[58.25]], [[59.5]], [[62.0]], [[52.5]], [[14.9375]], [[23.875]], [[24.625]], [[38.25]], [[53.75]], [[124.5]], [[32.0]], [[56.5]], [[65.5]], [[86.5]], [[70.0]], [[61.75]], [[42.25]], [[39.25]], [[83.5]], [[31.875]], [[46.75]], [[40.25]], [[22.75]], [[14.5]], [[41.75]], [[128.0]], [[17.375]], [[43.75]], [[57.25]], [[21.75]], [[18.75]], [[106.0]], [[23.25]], [[88.5]], [[25.625]], [[51.0]], [[22.0]], [[8.375]], [[58.5]], [[46.75]], [[19.5]], [[39.25]]]}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[64.0]]], "params": {"weight": [[0.423828125]]}}, "language_model.model.layers.33.feed_forward.shared_expert.down_proj": {"inputs": [[[79.5]]], "outputs": [[[21.75]], [[2.9110093107689658e+38]]], "params": {"weight": [[0.89453125]]}}, "language_model.model.layers.34.self_attn.qkv_proj": {"inputs": [[[54.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.34.self_attn.o_proj": {"inputs": [[[7.84375]]], "outputs": [[[46.75]], [[80.5]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.34.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.3203125]], [[18.25]]]}, "language_model.model.layers.34.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.6875]]]}, "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.3203125]]]}, "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.1875]]]}, "language_model.model.layers.34.self_attn.attn.impl.k_cache": {"inputs": [[[18.25]]]}, "language_model.model.layers.34.self_attn.attn.impl.v_cache": {"inputs": [[[9.6875]]]}, "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[17.0]], [[18.25]], [[9.6875]]], "outputs": [[[7.84375]], [[1.0]]]}, "language_model.model.layers.34.feed_forward.gate_up_proj": {"inputs": [[[31.125]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.34.feed_forward.down_proj": {"inputs": [[[86.0]]], "outputs": [[[30.375]], [[128.0]]], "params": {"weight": [[0.48828125]]}}, "language_model.model.layers.35.self_attn.qkv_proj": {"inputs": [[[52.75]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.35.self_attn.o_proj": {"inputs": [[[5.53125]]], "outputs": [[[87.5]], [[93.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.35.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.984375]], [[12.0625]]]}, "language_model.model.layers.35.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.34375]]]}, "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.984375]]]}, "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.53125]]]}, "language_model.model.layers.35.self_attn.attn.impl.k_cache": {"inputs": [[[12.0625]]]}, "language_model.model.layers.35.self_attn.attn.impl.v_cache": {"inputs": [[[7.34375]]]}, "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.25]], [[12.0625]], [[7.34375]]], "outputs": [[[5.5]], [[1.0]]]}, "language_model.model.layers.35.feed_forward.router": {"inputs": [[[64.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op": {"inputs": [[[64.0]]], "outputs": [[[16.625]], [[111.0]], [[33.5]], [[34.75]], [[36.5]], [[103.0]], [[20.875]], [[63.25]], [[99.5]], [[60.75]], [[75.0]], [[107.0]], [[25.75]], [[92.0]], [[41.75]], [[62.75]], [[60.5]], [[79.5]], [[40.25]], [[54.0]], [[51.5]], [[29.25]], [[37.75]], [[37.25]], [[38.5]], [[53.5]], [[47.25]], [[104.0]], [[51.0]], [[20.25]], [[63.25]], [[83.0]], [[55.25]], [[85.0]], [[32.25]], [[48.5]], [[78.5]], [[43.75]], [[46.5]], [[56.0]], [[38.0]], [[33.0]], [[61.25]], [[20.0]], [[44.75]], [[74.0]], [[42.75]], [[20.125]], [[22.375]], [[46.75]], [[94.0]], [[49.75]], [[46.75]], [[46.25]], [[28.0]], [[24.375]], [[34.25]], [[83.0]], [[31.5]], [[126.5]], [[65.0]], [[82.5]], [[30.5]], [[58.0]], [[19.25]], [[49.5]], [[20.5]], [[148.0]], [[52.25]], [[28.75]], [[137.0]], [[29.5]], [[127.5]], [[44.5]], [[63.0]], [[51.25]], [[35.0]], [[52.5]], [[75.0]], [[82.5]], [[102.5]], [[81.5]], [[105.5]], [[46.0]], [[58.5]], [[119.0]], [[123.0]], [[58.0]], [[33.5]], [[17.0]], [[37.5]], [[109.5]], [[53.25]], [[76.0]], [[110.0]], [[146.0]], [[72.0]], [[77.5]], [[28.875]], [[123.5]], [[35.0]], [[192.0]], [[26.625]], [[57.75]], [[22.875]], [[22.0]], [[44.25]], [[30.375]], [[29.125]], [[139.0]], [[71.5]], [[107.0]], [[48.75]], [[53.75]], [[43.25]], [[30.625]], [[50.25]], [[46.75]], [[22.75]], [[52.75]], [[85.5]], [[66.0]], [[24.25]], [[35.75]], [[81.5]], [[89.0]], [[29.0]], [[19.125]], [[34.25]]]}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.53125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.38671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[64.0]]], "params": {"weight": [[0.431640625]]}}, "language_model.model.layers.35.feed_forward.shared_expert.down_proj": {"inputs": [[[84.0]]], "outputs": [[[39.75]], [[2.9110093107689658e+38]]], "params": {"weight": [[0.796875]]}}, "language_model.model.layers.36.self_attn.qkv_proj": {"inputs": [[[56.75]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.36.self_attn.o_proj": {"inputs": [[[6.8125]]], "outputs": [[[27.75]], [[43.0]]], "params": {"weight": [[0.515625]]}}, "language_model.model.layers.36.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0546875]], [[16.75]]]}, "language_model.model.layers.36.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.25]]]}, "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0546875]]]}, "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.875]]]}, "language_model.model.layers.36.self_attn.attn.impl.k_cache": {"inputs": [[[16.75]]]}, "language_model.model.layers.36.self_attn.attn.impl.v_cache": {"inputs": [[[8.25]]]}, "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.125]], [[16.75]], [[8.25]]], "outputs": [[[6.8125]], [[1.0]]]}, "language_model.model.layers.36.feed_forward.gate_up_proj": {"inputs": [[[29.625]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.36.feed_forward.down_proj": {"inputs": [[[75.0]]], "outputs": [[[38.25]], [[76.5]]], "params": {"weight": [[0.439453125]]}}, "language_model.model.layers.37.self_attn.qkv_proj": {"inputs": [[[50.25]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.37.self_attn.o_proj": {"inputs": [[[8.0625]]], "outputs": [[[102.0]], [[105.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.37.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.3046875]], [[16.375]]]}, "language_model.model.layers.37.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.625]]]}, "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.3046875]]]}, "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.21875]]]}, "language_model.model.layers.37.self_attn.attn.impl.k_cache": {"inputs": [[[16.375]]]}, "language_model.model.layers.37.self_attn.attn.impl.v_cache": {"inputs": [[[8.875]]]}, "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.625]], [[16.375]], [[8.875]]], "outputs": [[[8.0625]], [[1.0]]]}, "language_model.model.layers.37.feed_forward.router": {"inputs": [[[70.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op": {"inputs": [[[70.0]]], "outputs": [[[17.375]], [[85.0]], [[67.0]], [[60.5]], [[66.0]], [[57.25]], [[44.5]], [[55.0]], [[112.5]], [[29.0]], [[144.0]], [[26.375]], [[36.75]], [[45.75]], [[70.5]], [[59.0]], [[86.5]], [[63.25]], [[44.5]], [[62.75]], [[92.0]], [[79.5]], [[66.0]], [[100.0]], [[70.0]], [[99.5]], [[67.5]], [[101.0]], [[100.0]], [[149.0]], [[65.0]], [[22.125]], [[90.5]], [[34.25]], [[20.5]], [[63.75]], [[65.5]], [[102.0]], [[159.0]], [[98.5]], [[72.0]], [[83.0]], [[30.0]], [[31.625]], [[46.0]], [[33.75]], [[91.5]], [[164.0]], [[22.625]], [[100.0]], [[16.75]], [[32.75]], [[73.5]], [[105.0]], [[56.25]], [[60.75]], [[93.0]], [[98.5]], [[71.0]], [[74.0]], [[90.0]], [[64.0]], [[50.25]], [[83.0]], [[29.375]], [[52.5]], [[37.0]], [[40.0]], [[65.5]], [[82.0]], [[34.5]], [[52.0]], [[48.25]], [[53.25]], [[21.375]], [[33.75]], [[82.0]], [[212.0]], [[16.125]], [[50.0]], [[85.5]], [[86.0]], [[37.75]], [[143.0]], [[26.625]], [[52.75]], [[49.5]], [[60.75]], [[90.0]], [[104.0]], [[211.0]], [[104.5]], [[64.5]], [[58.75]], [[84.0]], [[32.0]], [[89.0]], [[19.75]], [[26.0]], [[83.5]], [[97.0]], [[146.0]], [[48.5]], [[52.0]], [[27.0]], [[94.5]], [[67.0]], [[117.5]], [[22.625]], [[79.5]], [[45.5]], [[40.75]], [[52.0]], [[63.0]], [[86.0]], [[55.25]], [[67.5]], [[55.25]], [[101.0]], [[40.5]], [[61.5]], [[21.125]], [[43.0]], [[49.5]], [[49.25]], [[85.0]], [[54.25]], [[58.0]], [[61.25]]]}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.482421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.388671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[70.0]]], "params": {"weight": [[0.357421875]]}}, "language_model.model.layers.37.feed_forward.shared_expert.down_proj": {"inputs": [[[112.0]]], "outputs": [[[31.25]], [[2.9110093107689658e+38]]], "params": {"weight": [[0.75390625]]}}, "language_model.model.layers.38.self_attn.qkv_proj": {"inputs": [[[47.25]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.38.self_attn.o_proj": {"inputs": [[[6.65625]]], "outputs": [[[26.75]], [[56.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.38.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.2734375]], [[15.0625]]]}, "language_model.model.layers.38.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.9375]]]}, "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.2734375]]]}, "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.15625]]]}, "language_model.model.layers.38.self_attn.attn.impl.k_cache": {"inputs": [[[15.0625]]]}, "language_model.model.layers.38.self_attn.attn.impl.v_cache": {"inputs": [[[6.9375]]]}, "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[17.375]], [[15.0625]], [[6.9375]]], "outputs": [[[6.65625]], [[1.0]]]}, "language_model.model.layers.38.feed_forward.gate_up_proj": {"inputs": [[[31.125]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.38.feed_forward.down_proj": {"inputs": [[[123.0]]], "outputs": [[[24.75]], [[193.0]]], "params": {"weight": [[0.8046875]]}}, "language_model.model.layers.39.self_attn.qkv_proj": {"inputs": [[[49.75]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.39.self_attn.o_proj": {"inputs": [[[6.625]]], "outputs": [[[129.0]], [[134.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.39.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.99609375]], [[13.25]]]}, "language_model.model.layers.39.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.9375]]]}, "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.99609375]]]}, "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.25]]]}, "language_model.model.layers.39.self_attn.attn.impl.k_cache": {"inputs": [[[13.25]]]}, "language_model.model.layers.39.self_attn.attn.impl.v_cache": {"inputs": [[[9.9375]]]}, "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.5625]], [[13.25]], [[9.9375]]], "outputs": [[[6.625]], [[1.0]]]}, "language_model.model.layers.39.feed_forward.router": {"inputs": [[[61.5]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op": {"inputs": [[[61.5]]], "outputs": [[[12.8125]], [[52.25]], [[55.5]], [[103.0]], [[85.0]], [[63.25]], [[40.5]], [[48.0]], [[97.0]], [[39.0]], [[70.0]], [[77.0]], [[32.0]], [[95.5]], [[111.0]], [[72.0]], [[78.5]], [[92.0]], [[106.0]], [[52.0]], [[61.5]], [[51.75]], [[86.5]], [[51.25]], [[78.0]], [[76.5]], [[91.0]], [[86.0]], [[103.5]], [[76.5]], [[46.75]], [[60.25]], [[151.0]], [[26.25]], [[31.375]], [[97.5]], [[45.25]], [[151.0]], [[82.0]], [[116.5]], [[82.0]], [[91.0]], [[53.0]], [[130.0]], [[108.5]], [[58.75]], [[79.0]], [[59.0]], [[68.0]], [[89.0]], [[46.0]], [[125.0]], [[94.5]], [[103.5]], [[72.0]], [[47.25]], [[114.5]], [[31.625]], [[123.0]], [[41.25]], [[35.75]], [[131.0]], [[138.0]], [[88.0]], [[19.625]], [[41.0]], [[66.0]], [[34.25]], [[69.5]], [[116.0]], [[102.5]], [[97.5]], [[50.0]], [[74.5]], [[145.0]], [[70.0]], [[53.5]], [[105.0]], [[90.0]], [[119.5]], [[44.5]], [[48.75]], [[73.5]], [[77.0]], [[119.0]], [[75.5]], [[93.5]], [[128.0]], [[67.5]], [[94.5]], [[81.5]], [[41.0]], [[87.5]], [[59.5]], [[68.0]], [[37.75]], [[40.25]], [[51.25]], [[89.0]], [[65.0]], [[56.5]], [[41.0]], [[108.0]], [[85.5]], [[88.0]], [[49.75]], [[122.5]], [[81.5]], [[86.5]], [[59.25]], [[69.5]], [[39.0]], [[85.5]], [[40.0]], [[78.5]], [[76.5]], [[179.0]], [[28.625]], [[35.0]], [[106.5]], [[110.5]], [[27.875]], [[54.0]], [[96.5]], [[43.0]], [[79.0]], [[49.5]], [[101.0]], [[100.0]]]}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.32421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.419921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[61.5]]], "params": {"weight": [[0.447265625]]}}, "language_model.model.layers.39.feed_forward.shared_expert.down_proj": {"inputs": [[[137.0]]], "outputs": [[[22.125]], [[3.176854909925949e+38]]], "params": {"weight": [[0.419921875]]}}, "language_model.model.layers.40.self_attn.qkv_proj": {"inputs": [[[46.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.40.self_attn.o_proj": {"inputs": [[[9.9375]]], "outputs": [[[93.0]], [[90.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.40.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.2578125]], [[12.25]]]}, "language_model.model.layers.40.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[14.9375]]]}, "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.2578125]]]}, "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[10.0625]]]}, "language_model.model.layers.40.self_attn.attn.impl.k_cache": {"inputs": [[[12.25]]]}, "language_model.model.layers.40.self_attn.attn.impl.v_cache": {"inputs": [[[14.9375]]]}, "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.375]], [[12.25]], [[14.9375]]], "outputs": [[[9.9375]], [[1.0]]]}, "language_model.model.layers.40.feed_forward.gate_up_proj": {"inputs": [[[40.75]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.40.feed_forward.down_proj": {"inputs": [[[187.0]]], "outputs": [[[22.625]], [[146.0]]], "params": {"weight": [[0.62109375]]}}, "language_model.model.layers.41.self_attn.qkv_proj": {"inputs": [[[33.75]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.41.self_attn.o_proj": {"inputs": [[[5.40625]]], "outputs": [[[159.0]], [[164.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.41.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.140625]], [[14.4375]]]}, "language_model.model.layers.41.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.5625]]]}, "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.140625]]]}, "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.40625]]]}, "language_model.model.layers.41.self_attn.attn.impl.k_cache": {"inputs": [[[14.4375]]]}, "language_model.model.layers.41.self_attn.attn.impl.v_cache": {"inputs": [[[6.5625]]]}, "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.5625]], [[14.4375]], [[6.5625]]], "outputs": [[[4.75]], [[1.0]]]}, "language_model.model.layers.41.feed_forward.router": {"inputs": [[[68.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op": {"inputs": [[[68.0]]], "outputs": [[[27.0]], [[75.5]], [[37.75]], [[36.75]], [[106.5]], [[66.5]], [[59.75]], [[104.0]], [[89.5]], [[54.0]], [[81.0]], [[111.0]], [[49.25]], [[68.5]], [[95.5]], [[95.0]], [[88.0]], [[115.0]], [[127.0]], [[132.0]], [[41.75]], [[73.5]], [[40.75]], [[118.0]], [[90.5]], [[155.0]], [[132.0]], [[167.0]], [[61.5]], [[91.0]], [[79.0]], [[54.75]], [[67.5]], [[156.0]], [[52.25]], [[81.0]], [[100.5]], [[56.0]], [[106.0]], [[88.0]], [[73.0]], [[43.5]], [[120.5]], [[58.5]], [[54.0]], [[46.0]], [[71.5]], [[76.0]], [[80.0]], [[198.0]], [[117.5]], [[95.5]], [[75.5]], [[90.5]], [[134.0]], [[91.0]], [[81.5]], [[125.0]], [[59.75]], [[71.5]], [[68.5]], [[113.5]], [[115.5]], [[48.25]], [[83.5]], [[133.0]], [[31.625]], [[169.0]], [[76.0]], [[51.75]], [[141.0]], [[74.5]], [[28.5]], [[81.0]], [[34.0]], [[88.0]], [[79.0]], [[68.5]], [[60.25]], [[53.0]], [[104.5]], [[143.0]], [[142.0]], [[98.0]], [[63.0]], [[85.0]], [[59.0]], [[102.5]], [[137.0]], [[55.0]], [[75.5]], [[64.5]], [[175.0]], [[101.5]], [[73.5]], [[60.0]], [[103.5]], [[54.75]], [[71.5]], [[53.75]], [[108.0]], [[107.5]], [[75.0]], [[144.0]], [[79.0]], [[76.0]], [[91.0]], [[100.0]], [[95.5]], [[31.25]], [[89.5]], [[194.0]], [[99.5]], [[82.0]], [[61.25]], [[57.5]], [[131.0]], [[90.0]], [[304.0]], [[114.0]], [[135.0]], [[85.0]], [[75.5]], [[105.5]], [[44.0]], [[67.5]], [[123.0]], [[67.0]], [[111.5]]]}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.51171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.388671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[68.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.41.feed_forward.shared_expert.down_proj": {"inputs": [[[234.0]]], "outputs": [[[29.5]], [[2.9110093107689658e+38]]], "params": {"weight": [[0.734375]]}}, "language_model.model.layers.42.self_attn.qkv_proj": {"inputs": [[[38.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.42.self_attn.o_proj": {"inputs": [[[8.3125]]], "outputs": [[[50.5]], [[134.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.42.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1796875]], [[16.5]]]}, "language_model.model.layers.42.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.9375]]]}, "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1796875]]]}, "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[8.3125]]]}, "language_model.model.layers.42.self_attn.attn.impl.k_cache": {"inputs": [[[16.875]]]}, "language_model.model.layers.42.self_attn.attn.impl.v_cache": {"inputs": [[[8.9375]]]}, "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.5625]], [[16.875]], [[8.6875]]], "outputs": [[[6.90625]], [[1.0]]]}, "language_model.model.layers.42.feed_forward.gate_up_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.42.feed_forward.down_proj": {"inputs": [[[173.0]]], "outputs": [[[59.5]], [[338.0]]], "params": {"weight": [[0.50390625]]}}, "language_model.model.layers.43.self_attn.qkv_proj": {"inputs": [[[45.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.43.self_attn.o_proj": {"inputs": [[[12.75]]], "outputs": [[[94.5]], [[232.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.43.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0703125]], [[16.5]]]}, "language_model.model.layers.43.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.875]]]}, "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0703125]]]}, "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[10.375]]]}, "language_model.model.layers.43.self_attn.attn.impl.k_cache": {"inputs": [[[16.5]]]}, "language_model.model.layers.43.self_attn.attn.impl.v_cache": {"inputs": [[[12.875]]]}, "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[19.875]], [[16.5]], [[12.875]]], "outputs": [[[12.75]], [[1.0]]]}, "language_model.model.layers.43.feed_forward.router": {"inputs": [[[68.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op": {"inputs": [[[68.0]]], "outputs": [[[23.0]], [[103.0]], [[63.75]], [[69.5]], [[65.5]], [[149.0]], [[42.75]], [[56.25]], [[119.5]], [[88.5]], [[87.0]], [[130.0]], [[101.0]], [[64.0]], [[87.5]], [[123.5]], [[79.5]], [[76.5]], [[99.5]], [[194.0]], [[49.75]], [[179.0]], [[107.0]], [[165.0]], [[78.5]], [[153.0]], [[122.5]], [[140.0]], [[60.75]], [[90.0]], [[104.0]], [[42.5]], [[50.0]], [[62.25]], [[113.5]], [[80.0]], [[126.0]], [[118.5]], [[79.0]], [[71.5]], [[135.0]], [[69.5]], [[66.0]], [[200.0]], [[51.25]], [[128.0]], [[169.0]], [[69.0]], [[38.75]], [[82.0]], [[100.5]], [[49.25]], [[133.0]], [[164.0]], [[67.5]], [[153.0]], [[75.0]], [[35.0]], [[70.0]], [[86.0]], [[130.0]], [[42.25]], [[76.0]], [[92.5]], [[101.5]], [[124.5]], [[145.0]], [[150.0]], [[62.25]], [[206.0]], [[46.5]], [[102.0]], [[87.0]], [[83.5]], [[115.5]], [[45.75]], [[59.5]], [[163.0]], [[55.75]], [[121.0]], [[102.0]], [[105.5]], [[89.5]], [[155.0]], [[75.5]], [[82.0]], [[124.5]], [[52.0]], [[87.0]], [[62.75]], [[69.5]], [[119.5]], [[53.0]], [[108.5]], [[131.0]], [[53.0]], [[50.0]], [[49.5]], [[102.5]], [[48.5]], [[72.0]], [[103.0]], [[77.0]], [[69.5]], [[87.0]], [[202.0]], [[94.0]], [[73.5]], [[184.0]], [[74.5]], [[64.5]], [[124.5]], [[85.0]], [[112.0]], [[154.0]], [[36.25]], [[137.0]], [[119.0]], [[95.0]], [[140.0]], [[39.5]], [[62.25]], [[133.0]], [[66.0]], [[115.5]], [[97.0]], [[144.0]], [[81.0]], [[55.75]]]}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.4140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[68.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.43.feed_forward.shared_expert.down_proj": {"inputs": [[[220.0]]], "outputs": [[[29.125]], [[3.176854909925949e+38]]], "params": {"weight": [[0.5625]]}}, "language_model.model.layers.44.self_attn.qkv_proj": {"inputs": [[[28.625]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.44.self_attn.o_proj": {"inputs": [[[10.25]]], "outputs": [[[68.0]], [[153.0]]], "params": {"weight": [[1.2890625]]}}, "language_model.model.layers.44.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.7265625]], [[10.3125]]]}, "language_model.model.layers.44.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[10.625]]]}, "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.7265625]]]}, "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[9.3125]]]}, "language_model.model.layers.44.self_attn.attn.impl.k_cache": {"inputs": [[[10.3125]]]}, "language_model.model.layers.44.self_attn.attn.impl.v_cache": {"inputs": [[[10.625]]]}, "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[22.0]], [[10.3125]], [[10.625]]], "outputs": [[[10.25]], [[1.0]]]}, "language_model.model.layers.44.feed_forward.gate_up_proj": {"inputs": [[[41.5]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.44.feed_forward.down_proj": {"inputs": [[[270.0]]], "outputs": [[[63.25]], [[191.0]]], "params": {"weight": [[0.43359375]]}}, "language_model.model.layers.45.self_attn.qkv_proj": {"inputs": [[[27.375]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.45.self_attn.o_proj": {"inputs": [[[12.8125]]], "outputs": [[[169.0]], [[220.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.45.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.2421875]], [[15.5625]]]}, "language_model.model.layers.45.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[15.5]]]}, "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.2421875]]]}, "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[10.3125]]]}, "language_model.model.layers.45.self_attn.attn.impl.k_cache": {"inputs": [[[15.5625]]]}, "language_model.model.layers.45.self_attn.attn.impl.v_cache": {"inputs": [[[15.5]]]}, "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.5]], [[15.5625]], [[15.5]]], "outputs": [[[12.8125]], [[1.0]]]}, "language_model.model.layers.45.feed_forward.router": {"inputs": [[[51.75]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op": {"inputs": [[[51.75]]], "outputs": [[[15.0625]], [[78.0]], [[19.625]], [[43.0]], [[82.0]], [[0.0]], [[39.5]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[32.5]], [[23.5]], [[38.5]], [[51.5]], [[0.0]], [[0.0]], [[0.0]], [[70.5]], [[0.0]], [[0.0]], [[0.0]], [[23.25]], [[0.0]], [[37.25]], [[55.25]], [[29.125]], [[45.25]], [[37.75]], [[0.0]], [[19.25]], [[21.75]], [[25.875]], [[49.25]], [[49.0]], [[49.5]], [[0.0]], [[36.0]], [[23.375]], [[44.75]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[83.0]], [[0.0]], [[44.0]], [[0.0]], [[0.0]], [[0.0]], [[142.0]], [[0.0]], [[73.5]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[68.5]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[53.5]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[69.5]], [[34.0]], [[98.0]], [[30.25]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[98.5]], [[0.0]], [[144.0]], [[31.875]], [[45.5]], [[48.0]], [[0.0]], [[0.0]], [[30.75]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[17.5]], [[7.209944447028604e-18]], [[38.25]], [[0.0]], [[75.0]], [[0.0]], [[0.0]]]}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[51.75]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.45.feed_forward.shared_expert.down_proj": {"inputs": [[[224.0]]], "outputs": [[[122.5]], [[3.176854909925949e+38]]], "params": {"weight": [[0.70703125]]}}, "language_model.model.layers.46.self_attn.qkv_proj": {"inputs": [[[26.75]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.46.self_attn.o_proj": {"inputs": [[[32.5]]], "outputs": [[[101.0]], [[188.0]]], "params": {"weight": [[0.65234375]]}}, "language_model.model.layers.46.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.65625]], [[22.25]]]}, "language_model.model.layers.46.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[32.5]]]}, "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.65625]]]}, "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[26.75]]]}, "language_model.model.layers.46.self_attn.attn.impl.k_cache": {"inputs": [[[22.25]]]}, "language_model.model.layers.46.self_attn.attn.impl.v_cache": {"inputs": [[[32.5]]]}, "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[20.75]], [[21.375]], [[32.5]]], "outputs": [[[32.5]], [[1.0]]]}, "language_model.model.layers.46.feed_forward.gate_up_proj": {"inputs": [[[40.5]]], "params": {"weight": [[0.37109375]]}}, "language_model.model.layers.46.feed_forward.down_proj": {"inputs": [[[1456.0]]], "outputs": [[[1688.0]], [[2400.0]]], "params": {"weight": [[0.498046875]]}}, "language_model.model.layers.47.self_attn.qkv_proj": {"inputs": [[[19.875]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.47.self_attn.o_proj": {"inputs": [[[7.4375]]], "outputs": [[[98.0]], [[126.5]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.47.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8671875]], [[13.125]]]}, "language_model.model.layers.47.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[14.0]]]}, "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8671875]]]}, "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.96875]]]}, "language_model.model.layers.47.self_attn.attn.impl.k_cache": {"inputs": [[[13.125]]]}, "language_model.model.layers.47.self_attn.attn.impl.v_cache": {"inputs": [[[14.0]]]}, "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[5.75]], [[13.125]], [[14.0]]], "outputs": [[[7.15625]], [[1.0]]]}, "language_model.model.layers.47.feed_forward.router": {"inputs": [[[33.5]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op": {"inputs": [[[33.5]]], "outputs": [[[190.0]], [[52.25]], [[66.5]], [[39.5]], [[264.0]], [[84.5]], [[26.125]], [[54.5]], [[57.25]], [[69.5]], [[217.0]], [[114.5]], [[228.0]], [[114.0]], [[37.75]], [[143.0]], [[29.125]], [[49.25]], [[48.75]], [[63.75]], [[77.0]], [[155.0]], [[111.0]], [[116.0]], [[67.0]], [[350.0]], [[177.0]], [[32.5]], [[38.0]], [[40.5]], [[45.5]], [[34.5]], [[48.25]], [[23.875]], [[88.5]], [[69.0]], [[24.75]], [[41.25]], [[22.75]], [[199.0]], [[91.5]], [[32.25]], [[139.0]], [[206.0]], [[28.125]], [[53.25]], [[74.0]], [[88.5]], [[48.0]], [[25.25]], [[92.0]], [[102.5]], [[282.0]], [[78.0]], [[33.75]], [[34.25]], [[29.75]], [[138.0]], [[38.25]], [[35.75]], [[49.5]], [[53.25]], [[67.5]], [[134.0]], [[123.0]], [[70.5]], [[32.25]], [[22.25]], [[114.5]], [[52.0]], [[270.0]], [[264.0]], [[26.125]], [[79.0]], [[27.125]], [[93.0]], [[19.875]], [[236.0]], [[60.25]], [[258.0]], [[39.0]], [[127.0]], [[178.0]], [[93.0]], [[42.0]], [[42.25]], [[114.5]], [[114.0]], [[46.25]], [[40.75]], [[110.0]], [[97.0]], [[120.5]], [[312.0]], [[31.5]], [[35.5]], [[85.0]], [[100.5]], [[34.5]], [[24.875]], [[69.0]], [[59.0]], [[40.5]], [[78.5]], [[48.0]], [[150.0]], [[29.0]], [[254.0]], [[288.0]], [[36.25]], [[64.0]], [[83.0]], [[181.0]], [[43.25]], [[132.0]], [[107.0]], [[59.25]], [[154.0]], [[286.0]], [[35.5]], [[380.0]], [[316.0]], [[73.0]], [[100.5]], [[30.375]], [[35.0]], [[177.0]], [[155.0]], [[236.0]]]}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.7109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.76171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.45703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.498046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.380859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.3671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.451171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.81640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.7265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.498046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.4765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.84375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.396484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.35546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.83984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.51171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.4375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.50390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.8359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.43359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.35546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.6484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.5859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.462890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.404296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.69140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.357421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.8125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.73828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.37109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.4375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.408203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.486328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.76171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.7578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.416015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.57421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.470703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.400390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.5234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.93359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.384765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.59375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.73828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.384765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.419921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.392578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.56640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.51953125]]}}, "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[33.5]]], "params": {"weight": [[0.384765625]]}}, "language_model.model.layers.47.feed_forward.shared_expert.down_proj": {"inputs": [[[1112.0]]], "outputs": [[[720.0]], [[8.706443372391199e+37]]], "params": {"weight": [[0.408203125]]}}, "language_model.lm_head": {"inputs": [[[52.0]]], "params": {"weight": [[1.0234375]]}}}} \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_5_8.npz b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_5_8.npz new file mode 100644 index 000000000000..5f90f4c99769 Binary files /dev/null and b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_5_8.npz differ diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_5_8_mod_list.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_5_8_mod_list.json new file mode 100644 index 000000000000..928cd9e2a2c5 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_5_8_mod_list.json @@ -0,0 +1,6935 @@ +[ + "vision_model.patch_embedding.linear", + "vision_model.model.layers.0.self_attn.qkv_proj", + "vision_model.model.layers.0.self_attn.o_proj", + "vision_model.model.layers.0.mlp.fc1", + "vision_model.model.layers.0.mlp.fc2", + "vision_model.model.layers.1.self_attn.qkv_proj", + "vision_model.model.layers.1.self_attn.o_proj", + "vision_model.model.layers.1.mlp.fc1", + "vision_model.model.layers.1.mlp.fc2", + "vision_model.model.layers.2.self_attn.qkv_proj", + "vision_model.model.layers.2.self_attn.o_proj", + "vision_model.model.layers.2.mlp.fc1", + "vision_model.model.layers.2.mlp.fc2", + "vision_model.model.layers.3.self_attn.qkv_proj", + "vision_model.model.layers.3.self_attn.o_proj", + "vision_model.model.layers.3.mlp.fc1", + "vision_model.model.layers.3.mlp.fc2", + "vision_model.model.layers.4.self_attn.qkv_proj", + "vision_model.model.layers.4.self_attn.o_proj", + "vision_model.model.layers.4.mlp.fc1", + "vision_model.model.layers.4.mlp.fc2", + "vision_model.model.layers.5.self_attn.qkv_proj", + "vision_model.model.layers.5.self_attn.o_proj", + "vision_model.model.layers.5.mlp.fc1", + "vision_model.model.layers.5.mlp.fc2", + "vision_model.model.layers.6.self_attn.qkv_proj", + "vision_model.model.layers.6.self_attn.o_proj", + "vision_model.model.layers.6.mlp.fc1", + "vision_model.model.layers.6.mlp.fc2", + "vision_model.model.layers.7.self_attn.qkv_proj", + "vision_model.model.layers.7.self_attn.o_proj", + "vision_model.model.layers.7.mlp.fc1", + "vision_model.model.layers.7.mlp.fc2", + "vision_model.model.layers.8.self_attn.qkv_proj", + "vision_model.model.layers.8.self_attn.o_proj", + "vision_model.model.layers.8.mlp.fc1", + "vision_model.model.layers.8.mlp.fc2", + "vision_model.model.layers.9.self_attn.qkv_proj", + "vision_model.model.layers.9.self_attn.o_proj", + "vision_model.model.layers.9.mlp.fc1", + "vision_model.model.layers.9.mlp.fc2", + "vision_model.model.layers.10.self_attn.qkv_proj", + "vision_model.model.layers.10.self_attn.o_proj", + "vision_model.model.layers.10.mlp.fc1", + "vision_model.model.layers.10.mlp.fc2", + "vision_model.model.layers.11.self_attn.qkv_proj", + "vision_model.model.layers.11.self_attn.o_proj", + "vision_model.model.layers.11.mlp.fc1", + "vision_model.model.layers.11.mlp.fc2", + "vision_model.model.layers.12.self_attn.qkv_proj", + "vision_model.model.layers.12.self_attn.o_proj", + "vision_model.model.layers.12.mlp.fc1", + "vision_model.model.layers.12.mlp.fc2", + "vision_model.model.layers.13.self_attn.qkv_proj", + "vision_model.model.layers.13.self_attn.o_proj", + "vision_model.model.layers.13.mlp.fc1", + "vision_model.model.layers.13.mlp.fc2", + "vision_model.model.layers.14.self_attn.qkv_proj", + "vision_model.model.layers.14.self_attn.o_proj", + "vision_model.model.layers.14.mlp.fc1", + "vision_model.model.layers.14.mlp.fc2", + "vision_model.model.layers.15.self_attn.qkv_proj", + "vision_model.model.layers.15.self_attn.o_proj", + "vision_model.model.layers.15.mlp.fc1", + "vision_model.model.layers.15.mlp.fc2", + "vision_model.model.layers.16.self_attn.qkv_proj", + "vision_model.model.layers.16.self_attn.o_proj", + "vision_model.model.layers.16.mlp.fc1", + "vision_model.model.layers.16.mlp.fc2", + "vision_model.model.layers.17.self_attn.qkv_proj", + "vision_model.model.layers.17.self_attn.o_proj", + "vision_model.model.layers.17.mlp.fc1", + "vision_model.model.layers.17.mlp.fc2", + "vision_model.model.layers.18.self_attn.qkv_proj", + "vision_model.model.layers.18.self_attn.o_proj", + "vision_model.model.layers.18.mlp.fc1", + "vision_model.model.layers.18.mlp.fc2", + "vision_model.model.layers.19.self_attn.qkv_proj", + "vision_model.model.layers.19.self_attn.o_proj", + "vision_model.model.layers.19.mlp.fc1", + "vision_model.model.layers.19.mlp.fc2", + "vision_model.model.layers.20.self_attn.qkv_proj", + "vision_model.model.layers.20.self_attn.o_proj", + "vision_model.model.layers.20.mlp.fc1", + "vision_model.model.layers.20.mlp.fc2", + "vision_model.model.layers.21.self_attn.qkv_proj", + "vision_model.model.layers.21.self_attn.o_proj", + "vision_model.model.layers.21.mlp.fc1", + "vision_model.model.layers.21.mlp.fc2", + "vision_model.model.layers.22.self_attn.qkv_proj", + "vision_model.model.layers.22.self_attn.o_proj", + "vision_model.model.layers.22.mlp.fc1", + "vision_model.model.layers.22.mlp.fc2", + "vision_model.model.layers.23.self_attn.qkv_proj", + "vision_model.model.layers.23.self_attn.o_proj", + "vision_model.model.layers.23.mlp.fc1", + "vision_model.model.layers.23.mlp.fc2", + "vision_model.model.layers.24.self_attn.qkv_proj", + "vision_model.model.layers.24.self_attn.o_proj", + "vision_model.model.layers.24.mlp.fc1", + "vision_model.model.layers.24.mlp.fc2", + "vision_model.model.layers.25.self_attn.qkv_proj", + "vision_model.model.layers.25.self_attn.o_proj", + "vision_model.model.layers.25.mlp.fc1", + "vision_model.model.layers.25.mlp.fc2", + "vision_model.model.layers.26.self_attn.qkv_proj", + "vision_model.model.layers.26.self_attn.o_proj", + "vision_model.model.layers.26.mlp.fc1", + "vision_model.model.layers.26.mlp.fc2", + "vision_model.model.layers.27.self_attn.qkv_proj", + "vision_model.model.layers.27.self_attn.o_proj", + "vision_model.model.layers.27.mlp.fc1", + "vision_model.model.layers.27.mlp.fc2", + "vision_model.model.layers.28.self_attn.qkv_proj", + "vision_model.model.layers.28.self_attn.o_proj", + "vision_model.model.layers.28.mlp.fc1", + "vision_model.model.layers.28.mlp.fc2", + "vision_model.model.layers.29.self_attn.qkv_proj", + "vision_model.model.layers.29.self_attn.o_proj", + "vision_model.model.layers.29.mlp.fc1", + "vision_model.model.layers.29.mlp.fc2", + "vision_model.model.layers.30.self_attn.qkv_proj", + "vision_model.model.layers.30.self_attn.o_proj", + "vision_model.model.layers.30.mlp.fc1", + "vision_model.model.layers.30.mlp.fc2", + "vision_model.model.layers.31.self_attn.qkv_proj", + "vision_model.model.layers.31.self_attn.o_proj", + "vision_model.model.layers.31.mlp.fc1", + "vision_model.model.layers.31.mlp.fc2", + "vision_model.model.layers.32.self_attn.qkv_proj", + "vision_model.model.layers.32.self_attn.o_proj", + "vision_model.model.layers.32.mlp.fc1", + "vision_model.model.layers.32.mlp.fc2", + "vision_model.model.layers.33.self_attn.qkv_proj", + "vision_model.model.layers.33.self_attn.o_proj", + "vision_model.model.layers.33.mlp.fc1", + "vision_model.model.layers.33.mlp.fc2", + "vision_model.vision_adapter.mlp.fc1", + "vision_model.vision_adapter.mlp.fc2", + "multi_modal_projector.linear_1", + "language_model.model.layers.0.self_attn.qkv_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.0.self_attn.attn.impl.softmax", + "language_model.model.layers.0.self_attn.attn.impl.matmul_av", + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.0.self_attn.attn.impl.k_cache", + "language_model.model.layers.0.self_attn.attn.impl.v_cache", + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.0.feed_forward.gate_up_proj", + "language_model.model.layers.0.feed_forward.down_proj", + "language_model.model.layers.1.self_attn.qkv_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.1.self_attn.attn.impl.softmax", + "language_model.model.layers.1.self_attn.attn.impl.matmul_av", + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.1.self_attn.attn.impl.k_cache", + "language_model.model.layers.1.self_attn.attn.impl.v_cache", + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.1.feed_forward.router", + "language_model.model.layers.1.feed_forward.experts", + "language_model.model.layers.1.feed_forward.experts.moe_op", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.1.feed_forward.shared_expert.down_proj", + "language_model.model.layers.2.self_attn.qkv_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.2.self_attn.attn.impl.softmax", + "language_model.model.layers.2.self_attn.attn.impl.matmul_av", + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.2.self_attn.attn.impl.k_cache", + "language_model.model.layers.2.self_attn.attn.impl.v_cache", + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.2.feed_forward.gate_up_proj", + "language_model.model.layers.2.feed_forward.down_proj", + "language_model.model.layers.3.self_attn.qkv_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.3.self_attn.attn.impl.softmax", + "language_model.model.layers.3.self_attn.attn.impl.matmul_av", + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.3.self_attn.attn.impl.k_cache", + "language_model.model.layers.3.self_attn.attn.impl.v_cache", + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.3.feed_forward.router", + "language_model.model.layers.3.feed_forward.experts", + "language_model.model.layers.3.feed_forward.experts.moe_op", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.3.feed_forward.shared_expert.down_proj", + "language_model.model.layers.4.self_attn.qkv_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.4.self_attn.attn.impl.softmax", + "language_model.model.layers.4.self_attn.attn.impl.matmul_av", + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.4.self_attn.attn.impl.k_cache", + "language_model.model.layers.4.self_attn.attn.impl.v_cache", + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.4.feed_forward.gate_up_proj", + "language_model.model.layers.4.feed_forward.down_proj", + "language_model.model.layers.5.self_attn.qkv_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.5.self_attn.attn.impl.softmax", + "language_model.model.layers.5.self_attn.attn.impl.matmul_av", + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.5.self_attn.attn.impl.k_cache", + "language_model.model.layers.5.self_attn.attn.impl.v_cache", + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.5.feed_forward.router", + "language_model.model.layers.5.feed_forward.experts", + "language_model.model.layers.5.feed_forward.experts.moe_op", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.5.feed_forward.shared_expert.down_proj", + "language_model.model.layers.6.self_attn.qkv_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.6.self_attn.attn.impl.softmax", + "language_model.model.layers.6.self_attn.attn.impl.matmul_av", + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.6.self_attn.attn.impl.k_cache", + "language_model.model.layers.6.self_attn.attn.impl.v_cache", + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.6.feed_forward.gate_up_proj", + "language_model.model.layers.6.feed_forward.down_proj", + "language_model.model.layers.7.self_attn.qkv_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.7.self_attn.attn.impl.softmax", + "language_model.model.layers.7.self_attn.attn.impl.matmul_av", + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.7.self_attn.attn.impl.k_cache", + "language_model.model.layers.7.self_attn.attn.impl.v_cache", + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.7.feed_forward.router", + "language_model.model.layers.7.feed_forward.experts", + "language_model.model.layers.7.feed_forward.experts.moe_op", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.7.feed_forward.shared_expert.down_proj", + "language_model.model.layers.8.self_attn.qkv_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.8.self_attn.attn.impl.softmax", + "language_model.model.layers.8.self_attn.attn.impl.matmul_av", + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.8.self_attn.attn.impl.k_cache", + "language_model.model.layers.8.self_attn.attn.impl.v_cache", + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.8.feed_forward.gate_up_proj", + "language_model.model.layers.8.feed_forward.down_proj", + "language_model.model.layers.9.self_attn.qkv_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.9.self_attn.attn.impl.softmax", + "language_model.model.layers.9.self_attn.attn.impl.matmul_av", + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.9.self_attn.attn.impl.k_cache", + "language_model.model.layers.9.self_attn.attn.impl.v_cache", + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.9.feed_forward.router", + "language_model.model.layers.9.feed_forward.experts", + "language_model.model.layers.9.feed_forward.experts.moe_op", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.9.feed_forward.shared_expert.down_proj", + "language_model.model.layers.10.self_attn.qkv_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.10.self_attn.attn.impl.softmax", + "language_model.model.layers.10.self_attn.attn.impl.matmul_av", + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.10.self_attn.attn.impl.k_cache", + "language_model.model.layers.10.self_attn.attn.impl.v_cache", + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.10.feed_forward.gate_up_proj", + "language_model.model.layers.10.feed_forward.down_proj", + "language_model.model.layers.11.self_attn.qkv_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.11.self_attn.attn.impl.softmax", + "language_model.model.layers.11.self_attn.attn.impl.matmul_av", + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.11.self_attn.attn.impl.k_cache", + "language_model.model.layers.11.self_attn.attn.impl.v_cache", + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.11.feed_forward.router", + "language_model.model.layers.11.feed_forward.experts", + "language_model.model.layers.11.feed_forward.experts.moe_op", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.11.feed_forward.shared_expert.down_proj", + "language_model.model.layers.12.self_attn.qkv_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.12.self_attn.attn.impl.softmax", + "language_model.model.layers.12.self_attn.attn.impl.matmul_av", + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.12.self_attn.attn.impl.k_cache", + "language_model.model.layers.12.self_attn.attn.impl.v_cache", + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.12.feed_forward.gate_up_proj", + "language_model.model.layers.12.feed_forward.down_proj", + "language_model.model.layers.13.self_attn.qkv_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.13.self_attn.attn.impl.softmax", + "language_model.model.layers.13.self_attn.attn.impl.matmul_av", + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.13.self_attn.attn.impl.k_cache", + "language_model.model.layers.13.self_attn.attn.impl.v_cache", + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.13.feed_forward.router", + "language_model.model.layers.13.feed_forward.experts", + "language_model.model.layers.13.feed_forward.experts.moe_op", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.13.feed_forward.shared_expert.down_proj", + "language_model.model.layers.14.self_attn.qkv_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.14.self_attn.attn.impl.softmax", + "language_model.model.layers.14.self_attn.attn.impl.matmul_av", + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.14.self_attn.attn.impl.k_cache", + "language_model.model.layers.14.self_attn.attn.impl.v_cache", + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.14.feed_forward.gate_up_proj", + "language_model.model.layers.14.feed_forward.down_proj", + "language_model.model.layers.15.self_attn.qkv_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.15.self_attn.attn.impl.softmax", + "language_model.model.layers.15.self_attn.attn.impl.matmul_av", + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.15.self_attn.attn.impl.k_cache", + "language_model.model.layers.15.self_attn.attn.impl.v_cache", + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.15.feed_forward.router", + "language_model.model.layers.15.feed_forward.experts", + "language_model.model.layers.15.feed_forward.experts.moe_op", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.15.feed_forward.shared_expert.down_proj", + "language_model.model.layers.16.self_attn.qkv_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.16.self_attn.attn.impl.softmax", + "language_model.model.layers.16.self_attn.attn.impl.matmul_av", + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.16.self_attn.attn.impl.k_cache", + "language_model.model.layers.16.self_attn.attn.impl.v_cache", + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.16.feed_forward.gate_up_proj", + "language_model.model.layers.16.feed_forward.down_proj", + "language_model.model.layers.17.self_attn.qkv_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.17.self_attn.attn.impl.softmax", + "language_model.model.layers.17.self_attn.attn.impl.matmul_av", + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.17.self_attn.attn.impl.k_cache", + "language_model.model.layers.17.self_attn.attn.impl.v_cache", + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.17.feed_forward.router", + "language_model.model.layers.17.feed_forward.experts", + "language_model.model.layers.17.feed_forward.experts.moe_op", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.17.feed_forward.shared_expert.down_proj", + "language_model.model.layers.18.self_attn.qkv_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.18.self_attn.attn.impl.softmax", + "language_model.model.layers.18.self_attn.attn.impl.matmul_av", + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.18.self_attn.attn.impl.k_cache", + "language_model.model.layers.18.self_attn.attn.impl.v_cache", + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.18.feed_forward.gate_up_proj", + "language_model.model.layers.18.feed_forward.down_proj", + "language_model.model.layers.19.self_attn.qkv_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.19.self_attn.attn.impl.softmax", + "language_model.model.layers.19.self_attn.attn.impl.matmul_av", + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.19.self_attn.attn.impl.k_cache", + "language_model.model.layers.19.self_attn.attn.impl.v_cache", + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.19.feed_forward.router", + "language_model.model.layers.19.feed_forward.experts", + "language_model.model.layers.19.feed_forward.experts.moe_op", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.19.feed_forward.shared_expert.down_proj", + "language_model.model.layers.20.self_attn.qkv_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.20.self_attn.attn.impl.softmax", + "language_model.model.layers.20.self_attn.attn.impl.matmul_av", + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.20.self_attn.attn.impl.k_cache", + "language_model.model.layers.20.self_attn.attn.impl.v_cache", + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.20.feed_forward.gate_up_proj", + "language_model.model.layers.20.feed_forward.down_proj", + "language_model.model.layers.21.self_attn.qkv_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.21.self_attn.attn.impl.softmax", + "language_model.model.layers.21.self_attn.attn.impl.matmul_av", + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.21.self_attn.attn.impl.k_cache", + "language_model.model.layers.21.self_attn.attn.impl.v_cache", + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.21.feed_forward.router", + "language_model.model.layers.21.feed_forward.experts", + "language_model.model.layers.21.feed_forward.experts.moe_op", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.21.feed_forward.shared_expert.down_proj", + "language_model.model.layers.22.self_attn.qkv_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.22.self_attn.attn.impl.softmax", + "language_model.model.layers.22.self_attn.attn.impl.matmul_av", + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.22.self_attn.attn.impl.k_cache", + "language_model.model.layers.22.self_attn.attn.impl.v_cache", + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.22.feed_forward.gate_up_proj", + "language_model.model.layers.22.feed_forward.down_proj", + "language_model.model.layers.23.self_attn.qkv_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.23.self_attn.attn.impl.softmax", + "language_model.model.layers.23.self_attn.attn.impl.matmul_av", + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.23.self_attn.attn.impl.k_cache", + "language_model.model.layers.23.self_attn.attn.impl.v_cache", + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.23.feed_forward.router", + "language_model.model.layers.23.feed_forward.experts", + "language_model.model.layers.23.feed_forward.experts.moe_op", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.23.feed_forward.shared_expert.down_proj", + "language_model.model.layers.24.self_attn.qkv_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.24.self_attn.attn.impl.softmax", + "language_model.model.layers.24.self_attn.attn.impl.matmul_av", + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.24.self_attn.attn.impl.k_cache", + "language_model.model.layers.24.self_attn.attn.impl.v_cache", + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.24.feed_forward.gate_up_proj", + "language_model.model.layers.24.feed_forward.down_proj", + "language_model.model.layers.25.self_attn.qkv_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.25.self_attn.attn.impl.softmax", + "language_model.model.layers.25.self_attn.attn.impl.matmul_av", + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.25.self_attn.attn.impl.k_cache", + "language_model.model.layers.25.self_attn.attn.impl.v_cache", + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.25.feed_forward.router", + "language_model.model.layers.25.feed_forward.experts", + "language_model.model.layers.25.feed_forward.experts.moe_op", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.25.feed_forward.shared_expert.down_proj", + "language_model.model.layers.26.self_attn.qkv_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.26.self_attn.attn.impl.softmax", + "language_model.model.layers.26.self_attn.attn.impl.matmul_av", + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.26.self_attn.attn.impl.k_cache", + "language_model.model.layers.26.self_attn.attn.impl.v_cache", + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.26.feed_forward.gate_up_proj", + "language_model.model.layers.26.feed_forward.down_proj", + "language_model.model.layers.27.self_attn.qkv_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.27.self_attn.attn.impl.softmax", + "language_model.model.layers.27.self_attn.attn.impl.matmul_av", + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.27.self_attn.attn.impl.k_cache", + "language_model.model.layers.27.self_attn.attn.impl.v_cache", + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.27.feed_forward.router", + "language_model.model.layers.27.feed_forward.experts", + "language_model.model.layers.27.feed_forward.experts.moe_op", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.27.feed_forward.shared_expert.down_proj", + "language_model.model.layers.28.self_attn.qkv_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.28.self_attn.attn.impl.softmax", + "language_model.model.layers.28.self_attn.attn.impl.matmul_av", + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.28.self_attn.attn.impl.k_cache", + "language_model.model.layers.28.self_attn.attn.impl.v_cache", + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.28.feed_forward.gate_up_proj", + "language_model.model.layers.28.feed_forward.down_proj", + "language_model.model.layers.29.self_attn.qkv_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.29.self_attn.attn.impl.softmax", + "language_model.model.layers.29.self_attn.attn.impl.matmul_av", + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.29.self_attn.attn.impl.k_cache", + "language_model.model.layers.29.self_attn.attn.impl.v_cache", + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.29.feed_forward.router", + "language_model.model.layers.29.feed_forward.experts", + "language_model.model.layers.29.feed_forward.experts.moe_op", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.29.feed_forward.shared_expert.down_proj", + "language_model.model.layers.30.self_attn.qkv_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.30.self_attn.attn.impl.softmax", + "language_model.model.layers.30.self_attn.attn.impl.matmul_av", + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.30.self_attn.attn.impl.k_cache", + "language_model.model.layers.30.self_attn.attn.impl.v_cache", + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.30.feed_forward.gate_up_proj", + "language_model.model.layers.30.feed_forward.down_proj", + "language_model.model.layers.31.self_attn.qkv_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.31.self_attn.attn.impl.softmax", + "language_model.model.layers.31.self_attn.attn.impl.matmul_av", + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.31.self_attn.attn.impl.k_cache", + "language_model.model.layers.31.self_attn.attn.impl.v_cache", + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.31.feed_forward.router", + "language_model.model.layers.31.feed_forward.experts", + "language_model.model.layers.31.feed_forward.experts.moe_op", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.31.feed_forward.shared_expert.down_proj", + "language_model.model.layers.32.self_attn.qkv_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.32.self_attn.attn.impl.softmax", + "language_model.model.layers.32.self_attn.attn.impl.matmul_av", + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.32.self_attn.attn.impl.k_cache", + "language_model.model.layers.32.self_attn.attn.impl.v_cache", + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.32.feed_forward.gate_up_proj", + "language_model.model.layers.32.feed_forward.down_proj", + "language_model.model.layers.33.self_attn.qkv_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.33.self_attn.attn.impl.softmax", + "language_model.model.layers.33.self_attn.attn.impl.matmul_av", + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.33.self_attn.attn.impl.k_cache", + "language_model.model.layers.33.self_attn.attn.impl.v_cache", + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.33.feed_forward.router", + "language_model.model.layers.33.feed_forward.experts", + "language_model.model.layers.33.feed_forward.experts.moe_op", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.33.feed_forward.shared_expert.down_proj", + "language_model.model.layers.34.self_attn.qkv_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.34.self_attn.attn.impl.softmax", + "language_model.model.layers.34.self_attn.attn.impl.matmul_av", + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.34.self_attn.attn.impl.k_cache", + "language_model.model.layers.34.self_attn.attn.impl.v_cache", + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.34.feed_forward.gate_up_proj", + "language_model.model.layers.34.feed_forward.down_proj", + "language_model.model.layers.35.self_attn.qkv_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.35.self_attn.attn.impl.softmax", + "language_model.model.layers.35.self_attn.attn.impl.matmul_av", + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.35.self_attn.attn.impl.k_cache", + "language_model.model.layers.35.self_attn.attn.impl.v_cache", + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.35.feed_forward.router", + "language_model.model.layers.35.feed_forward.experts", + "language_model.model.layers.35.feed_forward.experts.moe_op", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.35.feed_forward.shared_expert.down_proj", + "language_model.model.layers.36.self_attn.qkv_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.36.self_attn.attn.impl.softmax", + "language_model.model.layers.36.self_attn.attn.impl.matmul_av", + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.36.self_attn.attn.impl.k_cache", + "language_model.model.layers.36.self_attn.attn.impl.v_cache", + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.36.feed_forward.gate_up_proj", + "language_model.model.layers.36.feed_forward.down_proj", + "language_model.model.layers.37.self_attn.qkv_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.37.self_attn.attn.impl.softmax", + "language_model.model.layers.37.self_attn.attn.impl.matmul_av", + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.37.self_attn.attn.impl.k_cache", + "language_model.model.layers.37.self_attn.attn.impl.v_cache", + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.37.feed_forward.router", + "language_model.model.layers.37.feed_forward.experts", + "language_model.model.layers.37.feed_forward.experts.moe_op", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.37.feed_forward.shared_expert.down_proj", + "language_model.model.layers.38.self_attn.qkv_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.38.self_attn.attn.impl.softmax", + "language_model.model.layers.38.self_attn.attn.impl.matmul_av", + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.38.self_attn.attn.impl.k_cache", + "language_model.model.layers.38.self_attn.attn.impl.v_cache", + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.38.feed_forward.gate_up_proj", + "language_model.model.layers.38.feed_forward.down_proj", + "language_model.model.layers.39.self_attn.qkv_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.39.self_attn.attn.impl.softmax", + "language_model.model.layers.39.self_attn.attn.impl.matmul_av", + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.39.self_attn.attn.impl.k_cache", + "language_model.model.layers.39.self_attn.attn.impl.v_cache", + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.39.feed_forward.router", + "language_model.model.layers.39.feed_forward.experts", + "language_model.model.layers.39.feed_forward.experts.moe_op", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.39.feed_forward.shared_expert.down_proj", + "language_model.model.layers.40.self_attn.qkv_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.40.self_attn.attn.impl.softmax", + "language_model.model.layers.40.self_attn.attn.impl.matmul_av", + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.40.self_attn.attn.impl.k_cache", + "language_model.model.layers.40.self_attn.attn.impl.v_cache", + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.40.feed_forward.gate_up_proj", + "language_model.model.layers.40.feed_forward.down_proj", + "language_model.model.layers.41.self_attn.qkv_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.41.self_attn.attn.impl.softmax", + "language_model.model.layers.41.self_attn.attn.impl.matmul_av", + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.41.self_attn.attn.impl.k_cache", + "language_model.model.layers.41.self_attn.attn.impl.v_cache", + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.41.feed_forward.router", + "language_model.model.layers.41.feed_forward.experts", + "language_model.model.layers.41.feed_forward.experts.moe_op", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.41.feed_forward.shared_expert.down_proj", + "language_model.model.layers.42.self_attn.qkv_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.42.self_attn.attn.impl.softmax", + "language_model.model.layers.42.self_attn.attn.impl.matmul_av", + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.42.self_attn.attn.impl.k_cache", + "language_model.model.layers.42.self_attn.attn.impl.v_cache", + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.42.feed_forward.gate_up_proj", + "language_model.model.layers.42.feed_forward.down_proj", + "language_model.model.layers.43.self_attn.qkv_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.43.self_attn.attn.impl.softmax", + "language_model.model.layers.43.self_attn.attn.impl.matmul_av", + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.43.self_attn.attn.impl.k_cache", + "language_model.model.layers.43.self_attn.attn.impl.v_cache", + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.43.feed_forward.router", + "language_model.model.layers.43.feed_forward.experts", + "language_model.model.layers.43.feed_forward.experts.moe_op", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.43.feed_forward.shared_expert.down_proj", + "language_model.model.layers.44.self_attn.qkv_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.44.self_attn.attn.impl.softmax", + "language_model.model.layers.44.self_attn.attn.impl.matmul_av", + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.44.self_attn.attn.impl.k_cache", + "language_model.model.layers.44.self_attn.attn.impl.v_cache", + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.44.feed_forward.gate_up_proj", + "language_model.model.layers.44.feed_forward.down_proj", + "language_model.model.layers.45.self_attn.qkv_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.45.self_attn.attn.impl.softmax", + "language_model.model.layers.45.self_attn.attn.impl.matmul_av", + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.45.self_attn.attn.impl.k_cache", + "language_model.model.layers.45.self_attn.attn.impl.v_cache", + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.45.feed_forward.router", + "language_model.model.layers.45.feed_forward.experts", + "language_model.model.layers.45.feed_forward.experts.moe_op", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.45.feed_forward.shared_expert.down_proj", + "language_model.model.layers.46.self_attn.qkv_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.46.self_attn.attn.impl.softmax", + "language_model.model.layers.46.self_attn.attn.impl.matmul_av", + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.46.self_attn.attn.impl.k_cache", + "language_model.model.layers.46.self_attn.attn.impl.v_cache", + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.46.feed_forward.gate_up_proj", + "language_model.model.layers.46.feed_forward.down_proj", + "language_model.model.layers.47.self_attn.qkv_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.47.self_attn.attn.impl.softmax", + "language_model.model.layers.47.self_attn.attn.impl.matmul_av", + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.47.self_attn.attn.impl.k_cache", + "language_model.model.layers.47.self_attn.attn.impl.v_cache", + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.47.feed_forward.router", + "language_model.model.layers.47.feed_forward.experts", + "language_model.model.layers.47.feed_forward.experts.moe_op", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.47.feed_forward.shared_expert.down_proj", + "language_model.lm_head" +] \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_6_8.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_6_8.json new file mode 100644 index 000000000000..126cd640d4b7 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_6_8.json @@ -0,0 +1 @@ +{"GlobalRank": null, "LocalRank": 6, "Mode": "DynamicRange", "Nodes": {"language_model.model.layers.0.self_attn.qkv_proj": {"inputs": [[[11.1875]]], "params": {"weight": [[0.431640625]]}}, "language_model.model.layers.0.self_attn.o_proj": {"inputs": [[[1.171875]]], "outputs": [[[0.80078125]], [[3.0]]], "params": {"weight": [[0.396484375]]}}, "language_model.model.layers.0.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.4140625]], [[4.5625]]]}, "language_model.model.layers.0.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.328125]]]}, "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.4140625]]]}, "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.328125]]]}, "language_model.model.layers.0.self_attn.attn.impl.k_cache": {"inputs": [[[4.5625]]]}, "language_model.model.layers.0.self_attn.attn.impl.v_cache": {"inputs": [[[1.328125]]]}, "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[4.9375]], [[4.5625]], [[1.328125]]], "outputs": [[[1.171875]], [[1.0]]]}, "language_model.model.layers.0.feed_forward.gate_up_proj": {"inputs": [[[2.546875]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.0.feed_forward.down_proj": {"inputs": [[[3.671875]]], "outputs": [[[26.75]], [[65.0]]], "params": {"weight": [[0.66015625]]}}, "language_model.model.layers.1.self_attn.qkv_proj": {"inputs": [[[12.3125]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.1.self_attn.o_proj": {"inputs": [[[0.453125]]], "outputs": [[[0.52734375]], [[1.0078125]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.1.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.7578125]], [[7.1875]]]}, "language_model.model.layers.1.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.87109375]]]}, "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.7578125]]]}, "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.48046875]]]}, "language_model.model.layers.1.self_attn.attn.impl.k_cache": {"inputs": [[[7.1875]]]}, "language_model.model.layers.1.self_attn.attn.impl.v_cache": {"inputs": [[[0.87109375]]]}, "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.375]], [[7.1875]], [[0.81640625]]], "outputs": [[[0.423828125]], [[1.0]]]}, "language_model.model.layers.1.feed_forward.router": {"inputs": [[[1.8828125]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op": {"inputs": [[[1.8828125]]], "outputs": [[[1.140625]], [[16.375]], [[0.0]], [[1.1875]], [[11.6875]], [[10.75]], [[12.75]], [[10.125]], [[8.25]], [[8.625]], [[9.6875]], [[10.1875]], [[11.3125]], [[1.7578125]], [[8.1875]], [[14.25]], [[20.5]], [[9.5625]], [[4.375]], [[8.75]], [[6.90625]], [[15.3125]], [[11.25]], [[0.0]], [[15.5625]], [[8.1875]], [[0.60546875]], [[11.5]], [[16.0]], [[7.15625]], [[8.3125]], [[22.375]], [[14.5625]], [[0.1240234375]], [[4.375]], [[0.0]], [[13.4375]], [[2.5]], [[14.1875]], [[8.6875]], [[13.5625]], [[14.0625]], [[17.0]], [[9.25]], [[9.9375]], [[2.46875]], [[13.25]], [[9.6875]], [[8.75]], [[2.953125]], [[0.0]], [[11.4375]], [[14.125]], [[4.28125]], [[6.375]], [[11.25]], [[15.9375]], [[10.5625]], [[0.470703125]], [[12.25]], [[8.9375]], [[9.5625]], [[10.25]], [[14.875]], [[0.0]], [[9.25]], [[3.625]], [[6.75]], [[11.4375]], [[6.53125]], [[8.3125]], [[12.625]], [[21.625]], [[10.75]], [[16.875]], [[11.125]], [[13.9375]], [[11.0]], [[11.0625]], [[5.375]], [[13.0]], [[0.0]], [[0.88671875]], [[14.75]], [[3.71875]], [[16.375]], [[14.5]], [[10.5]], [[9.0]], [[7.40625]], [[10.0]], [[9.1875]], [[9.375]], [[12.5625]], [[0.0]], [[11.5]], [[10.5]], [[0.1259765625]], [[0.0]], [[14.625]], [[14.6875]], [[12.75]], [[5.4375]], [[14.0625]], [[5.875]], [[0.99609375]], [[11.6875]], [[0.087890625]], [[0.78515625]], [[2.234375]], [[1.2734375]], [[7.875]], [[0.46875]], [[13.25]], [[0.0]], [[14.375]], [[15.6875]], [[16.125]], [[6.84375]], [[14.625]], [[2.828125]], [[11.1875]], [[9.25]], [[12.0]], [[5.5625]], [[14.0625]], [[8.9375]], [[14.3125]], [[17.0]]]}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.462890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0751953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.07373046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[1.8828125]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.1.feed_forward.shared_expert.down_proj": {"inputs": [[[4.3125]]], "outputs": [[[0.431640625]], [[2.5654100318648876e+38]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.2.self_attn.qkv_proj": {"inputs": [[[9.875]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.2.self_attn.o_proj": {"inputs": [[[0.734375]]], "outputs": [[[0.6875]], [[1.328125]]], "params": {"weight": [[0.361328125]]}}, "language_model.model.layers.2.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.58984375]], [[9.3125]]]}, "language_model.model.layers.2.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.234375]]]}, "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.58984375]]]}, "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.94921875]]]}, "language_model.model.layers.2.self_attn.attn.impl.k_cache": {"inputs": [[[9.3125]]]}, "language_model.model.layers.2.self_attn.attn.impl.v_cache": {"inputs": [[[1.234375]]]}, "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.4375]], [[9.3125]], [[1.234375]]], "outputs": [[[0.734375]], [[1.0]]]}, "language_model.model.layers.2.feed_forward.gate_up_proj": {"inputs": [[[6.125]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.2.feed_forward.down_proj": {"inputs": [[[6.53125]]], "outputs": [[[4.75]], [[25.25]]], "params": {"weight": [[0.578125]]}}, "language_model.model.layers.3.self_attn.qkv_proj": {"inputs": [[[7.21875]]], "params": {"weight": [[0.435546875]]}}, "language_model.model.layers.3.self_attn.o_proj": {"inputs": [[[0.2099609375]]], "outputs": [[[0.59765625]], [[2.40625]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.3.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.2060546875]], [[3.40625]]]}, "language_model.model.layers.3.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.283203125]]]}, "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.2060546875]]]}, "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.1748046875]]]}, "language_model.model.layers.3.self_attn.attn.impl.k_cache": {"inputs": [[[3.40625]]]}, "language_model.model.layers.3.self_attn.attn.impl.v_cache": {"inputs": [[[0.283203125]]]}, "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[2.609375]], [[3.390625]], [[0.283203125]]], "outputs": [[[0.2099609375]], [[1.0]]]}, "language_model.model.layers.3.feed_forward.router": {"inputs": [[[8.75]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op": {"inputs": [[[8.75]]], "outputs": [[[176.0]], [[1.2578125]], [[9.125]], [[24.125]], [[5.25]], [[0.0]], [[3.625]], [[18.875]], [[19.375]], [[14.4375]], [[10.375]], [[9.6875]], [[15.9375]], [[15.875]], [[16.375]], [[12.125]], [[11.3125]], [[2.4375]], [[2.421875]], [[7.375]], [[9.1875]], [[8.0625]], [[14.5625]], [[16.25]], [[14.625]], [[8.375]], [[2.890625]], [[11.9375]], [[9.125]], [[9.5625]], [[8.0]], [[13.0625]], [[6.78125]], [[10.625]], [[5.1875]], [[11.0]], [[15.0625]], [[4.0625]], [[1.0703125]], [[16.375]], [[5.875]], [[8.8125]], [[14.1875]], [[0.8359375]], [[6.28125]], [[17.25]], [[5.90625]], [[13.125]], [[10.125]], [[6.9375]], [[15.5625]], [[15.8125]], [[15.625]], [[13.4375]], [[2.171875]], [[13.625]], [[18.125]], [[2.453125]], [[11.5625]], [[11.5625]], [[6.84375]], [[12.75]], [[14.75]], [[13.875]], [[0.185546875]], [[14.6875]], [[9.4375]], [[11.5]], [[0.22265625]], [[2.984375]], [[0.0]], [[17.75]], [[7.03125]], [[11.75]], [[5.5625]], [[2.734375]], [[5.6875]], [[12.0]], [[0.12353515625]], [[9.6875]], [[13.875]], [[13.9375]], [[0.0]], [[4.875]], [[21.125]], [[7.84375]], [[10.5625]], [[16.25]], [[3.890625]], [[12.625]], [[22.625]], [[13.0625]], [[6.8125]], [[5.875]], [[13.6875]], [[0.0]], [[16.5]], [[14.75]], [[10.75]], [[9.8125]], [[1.2109375]], [[15.6875]], [[1.15625]], [[3.09375]], [[15.5625]], [[0.0]], [[2.6875]], [[16.875]], [[12.1875]], [[19.375]], [[10.25]], [[16.625]], [[3.140625]], [[15.4375]], [[0.87109375]], [[1.5703125]], [[4.125]], [[10.875]], [[0.96875]], [[3.84375]], [[4.15625]], [[13.875]], [[6.1875]], [[15.625]], [[13.125]], [[3.234375]], [[15.375]], [[8.9375]], [[14.25]]]}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0751953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.07421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.3828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.07470703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[8.75]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.3.feed_forward.shared_expert.down_proj": {"inputs": [[[52.0]]], "outputs": [[[42.5]], [[1.42227395548986e+38]]], "params": {"weight": [[0.81640625]]}}, "language_model.model.layers.4.self_attn.qkv_proj": {"inputs": [[[13.75]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.4.self_attn.o_proj": {"inputs": [[[2.09375]]], "outputs": [[[1.015625]], [[2.046875]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.4.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.71875]], [[10.125]]]}, "language_model.model.layers.4.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.25]]]}, "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.71875]]]}, "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.8515625]]]}, "language_model.model.layers.4.self_attn.attn.impl.k_cache": {"inputs": [[[10.125]]]}, "language_model.model.layers.4.self_attn.attn.impl.v_cache": {"inputs": [[[2.28125]]]}, "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.125]], [[10.125]], [[2.28125]]], "outputs": [[[2.09375]], [[1.0]]]}, "language_model.model.layers.4.feed_forward.gate_up_proj": {"inputs": [[[7.09375]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.4.feed_forward.down_proj": {"inputs": [[[6.90625]]], "outputs": [[[2.796875]], [[4.96875]]], "params": {"weight": [[0.419921875]]}}, "language_model.model.layers.5.self_attn.qkv_proj": {"inputs": [[[17.375]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.5.self_attn.o_proj": {"inputs": [[[3.0625]]], "outputs": [[[0.97265625]], [[2.21875]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.5.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.57421875]], [[12.6875]]]}, "language_model.model.layers.5.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.640625]]]}, "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.57421875]]]}, "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.3125]]]}, "language_model.model.layers.5.self_attn.attn.impl.k_cache": {"inputs": [[[12.6875]]]}, "language_model.model.layers.5.self_attn.attn.impl.v_cache": {"inputs": [[[3.640625]]]}, "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[6.75]], [[12.6875]], [[3.578125]]], "outputs": [[[3.0625]], [[1.0]]]}, "language_model.model.layers.5.feed_forward.router": {"inputs": [[[19.25]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op": {"inputs": [[[19.25]]], "outputs": [[[85.5]], [[20.0]], [[2.09375]], [[7.84375]], [[13.0]], [[11.875]], [[6.78125]], [[8.25]], [[16.125]], [[6.40625]], [[13.0625]], [[6.0625]], [[13.875]], [[17.5]], [[7.5625]], [[20.375]], [[19.875]], [[11.625]], [[5.1875]], [[19.5]], [[10.6875]], [[21.875]], [[9.375]], [[14.625]], [[21.25]], [[3.234375]], [[3.015625]], [[6.125]], [[84.5]], [[14.0625]], [[25.125]], [[25.625]], [[5.59375]], [[13.0]], [[27.875]], [[6.6875]], [[4.59375]], [[16.875]], [[3.546875]], [[12.8125]], [[22.125]], [[11.875]], [[3.234375]], [[11.875]], [[8.8125]], [[16.75]], [[12.0625]], [[12.6875]], [[12.375]], [[3.453125]], [[16.25]], [[11.8125]], [[18.875]], [[1.03125]], [[3.875]], [[14.4375]], [[3.28125]], [[10.8125]], [[15.3125]], [[14.0]], [[1.1015625]], [[7.78125]], [[15.6875]], [[3.40625]], [[19.75]], [[4.3125]], [[17.0]], [[5.28125]], [[7.09375]], [[19.625]], [[8.875]], [[4.375]], [[14.5625]], [[20.25]], [[15.5625]], [[4.84375]], [[22.25]], [[19.875]], [[14.4375]], [[21.25]], [[19.625]], [[1.375]], [[13.625]], [[12.1875]], [[11.625]], [[6.4375]], [[16.75]], [[0.1982421875]], [[13.9375]], [[26.125]], [[19.0]], [[18.375]], [[8.1875]], [[10.0625]], [[14.8125]], [[15.875]], [[5.03125]], [[15.4375]], [[4.65625]], [[3.90625]], [[16.625]], [[8.8125]], [[2.328125]], [[2.34375]], [[19.75]], [[12.875]], [[9.8125]], [[11.0625]], [[20.625]], [[9.375]], [[18.75]], [[12.875]], [[6.25]], [[14.0]], [[14.625]], [[18.375]], [[17.75]], [[5.0625]], [[8.375]], [[0.9453125]], [[7.84375]], [[15.0625]], [[18.625]], [[5.78125]], [[16.625]], [[14.375]], [[2.890625]], [[10.75]], [[8.5]]]}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.77734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.69140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[19.25]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.5.feed_forward.shared_expert.down_proj": {"inputs": [[[25.25]]], "outputs": [[[1.984375]], [[2.924301590726815e+38]]], "params": {"weight": [[0.376953125]]}}, "language_model.model.layers.6.self_attn.qkv_proj": {"inputs": [[[12.9375]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.6.self_attn.o_proj": {"inputs": [[[1.625]]], "outputs": [[[0.66796875]], [[1.5234375]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.6.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8359375]], [[16.125]]]}, "language_model.model.layers.6.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.484375]]]}, "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8359375]]]}, "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.765625]]]}, "language_model.model.layers.6.self_attn.attn.impl.k_cache": {"inputs": [[[16.125]]]}, "language_model.model.layers.6.self_attn.attn.impl.v_cache": {"inputs": [[[2.546875]]]}, "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.375]], [[16.125]], [[2.546875]]], "outputs": [[[1.625]], [[1.0]]]}, "language_model.model.layers.6.feed_forward.gate_up_proj": {"inputs": [[[7.71875]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.6.feed_forward.down_proj": {"inputs": [[[8.0625]]], "outputs": [[[2.328125]], [[11.1875]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.7.self_attn.qkv_proj": {"inputs": [[[7.09375]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.7.self_attn.o_proj": {"inputs": [[[0.5703125]]], "outputs": [[[1.84375]], [[3.84375]]], "params": {"weight": [[0.60546875]]}}, "language_model.model.layers.7.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.326171875]], [[6.1875]]]}, "language_model.model.layers.7.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.4453125]]]}, "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.326171875]]]}, "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.1484375]]]}, "language_model.model.layers.7.self_attn.attn.impl.k_cache": {"inputs": [[[6.375]]]}, "language_model.model.layers.7.self_attn.attn.impl.v_cache": {"inputs": [[[1.4453125]]]}, "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[4.28125]], [[6.375]], [[1.4453125]]], "outputs": [[[0.5703125]], [[1.0]]]}, "language_model.model.layers.7.feed_forward.router": {"inputs": [[[11.6875]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op": {"inputs": [[[11.6875]]], "outputs": [[[16.375]], [[11.4375]], [[15.5625]], [[21.25]], [[17.625]], [[7.75]], [[21.625]], [[9.5625]], [[12.375]], [[8.5625]], [[13.875]], [[17.5]], [[8.6875]], [[27.75]], [[15.875]], [[7.90625]], [[2.234375]], [[14.0625]], [[7.3125]], [[21.25]], [[18.625]], [[13.1875]], [[26.0]], [[16.75]], [[19.75]], [[26.75]], [[16.0]], [[7.4375]], [[8.125]], [[13.625]], [[14.0]], [[11.875]], [[16.5]], [[20.0]], [[4.53125]], [[9.875]], [[11.375]], [[4.625]], [[24.5]], [[22.5]], [[1.328125]], [[1.9609375]], [[11.5]], [[1.8125]], [[24.625]], [[10.125]], [[27.875]], [[0.5390625]], [[44.5]], [[17.875]], [[13.1875]], [[19.125]], [[10.6875]], [[14.8125]], [[6.78125]], [[20.25]], [[19.5]], [[13.4375]], [[2.90625]], [[8.75]], [[4.0625]], [[6.125]], [[23.0]], [[7.9375]], [[10.4375]], [[18.875]], [[16.0]], [[10.75]], [[26.125]], [[9.9375]], [[1.5234375]], [[29.75]], [[3.328125]], [[7.625]], [[11.25]], [[23.125]], [[5.0]], [[7.75]], [[6.875]], [[2.125]], [[15.375]], [[22.5]], [[22.375]], [[4.90625]], [[15.75]], [[6.71875]], [[18.375]], [[1.7421875]], [[19.5]], [[7.59375]], [[32.25]], [[12.75]], [[14.6875]], [[11.0]], [[2.5]], [[32.0]], [[6.90625]], [[19.375]], [[6.28125]], [[23.875]], [[13.625]], [[14.3125]], [[15.5625]], [[4.46875]], [[0.4375]], [[31.125]], [[21.125]], [[14.6875]], [[8.625]], [[1.6953125]], [[27.0]], [[16.75]], [[17.625]], [[8.8125]], [[43.75]], [[19.25]], [[2.828125]], [[3.90625]], [[16.75]], [[3.84375]], [[23.125]], [[24.5]], [[19.5]], [[19.75]], [[5.59375]], [[4.90625]], [[2.265625]], [[44.75]], [[12.375]]]}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.61328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[11.6875]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.7.feed_forward.shared_expert.down_proj": {"inputs": [[[9.875]]], "outputs": [[[0.5703125]], [[9.969209968386869e+37]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.8.self_attn.qkv_proj": {"inputs": [[[15.125]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.8.self_attn.o_proj": {"inputs": [[[1.9140625]]], "outputs": [[[1.9375]], [[2.109375]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.8.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.90625]], [[15.875]]]}, "language_model.model.layers.8.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.5]]]}, "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.90625]]]}, "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.984375]]]}, "language_model.model.layers.8.self_attn.attn.impl.k_cache": {"inputs": [[[15.875]]]}, "language_model.model.layers.8.self_attn.attn.impl.v_cache": {"inputs": [[[2.5]]]}, "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.5]], [[15.875]], [[2.5]]], "outputs": [[[1.9140625]], [[1.0]]]}, "language_model.model.layers.8.feed_forward.gate_up_proj": {"inputs": [[[8.25]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.8.feed_forward.down_proj": {"inputs": [[[9.5625]]], "outputs": [[[2.5625]], [[9.25]]], "params": {"weight": [[0.5]]}}, "language_model.model.layers.9.self_attn.qkv_proj": {"inputs": [[[15.75]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.9.self_attn.o_proj": {"inputs": [[[1.859375]]], "outputs": [[[1.3125]], [[2.875]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.9.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.89453125]], [[15.5]]]}, "language_model.model.layers.9.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.03125]]]}, "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.89453125]]]}, "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.21875]]]}, "language_model.model.layers.9.self_attn.attn.impl.k_cache": {"inputs": [[[15.5]]]}, "language_model.model.layers.9.self_attn.attn.impl.v_cache": {"inputs": [[[3.03125]]]}, "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.5]], [[15.5]], [[3.03125]]], "outputs": [[[1.859375]], [[1.0]]]}, "language_model.model.layers.9.feed_forward.router": {"inputs": [[[11.75]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op": {"inputs": [[[11.75]]], "outputs": [[[3.625]], [[12.9375]], [[11.625]], [[1.4609375]], [[35.75]], [[31.25]], [[19.5]], [[26.0]], [[18.875]], [[12.375]], [[33.75]], [[22.75]], [[10.5]], [[39.75]], [[30.0]], [[56.0]], [[3.40625]], [[22.5]], [[30.0]], [[54.25]], [[30.25]], [[20.875]], [[23.125]], [[29.375]], [[20.25]], [[14.0625]], [[23.625]], [[30.875]], [[21.0]], [[25.375]], [[69.0]], [[16.875]], [[15.4375]], [[32.0]], [[49.0]], [[45.0]], [[6.34375]], [[36.0]], [[24.625]], [[3.5]], [[18.75]], [[8.0625]], [[11.375]], [[20.125]], [[24.625]], [[11.0625]], [[10.375]], [[16.5]], [[15.75]], [[20.0]], [[19.375]], [[13.375]], [[24.25]], [[11.75]], [[10.1875]], [[21.5]], [[9.5]], [[26.125]], [[17.75]], [[34.25]], [[30.875]], [[26.125]], [[12.375]], [[22.125]], [[34.0]], [[9.4375]], [[24.125]], [[27.875]], [[22.375]], [[7.40625]], [[20.125]], [[38.0]], [[52.25]], [[9.25]], [[10.1875]], [[4.9375]], [[22.125]], [[11.75]], [[33.75]], [[6.375]], [[7.0625]], [[28.5]], [[32.75]], [[24.0]], [[20.875]], [[18.875]], [[26.125]], [[11.375]], [[17.25]], [[27.375]], [[32.75]], [[5.46875]], [[8.1875]], [[26.25]], [[15.9375]], [[28.25]], [[18.0]], [[6.875]], [[10.125]], [[21.875]], [[17.375]], [[18.25]], [[10.75]], [[40.5]], [[12.875]], [[20.125]], [[14.0]], [[27.25]], [[34.5]], [[25.25]], [[26.0]], [[14.875]], [[4.375]], [[21.625]], [[8.125]], [[51.0]], [[38.0]], [[15.375]], [[5.28125]], [[14.75]], [[11.5]], [[22.5]], [[9.875]], [[26.125]], [[27.0]], [[44.75]], [[19.25]], [[10.1875]], [[10.0]]]}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.50390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[11.75]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.9.feed_forward.shared_expert.down_proj": {"inputs": [[[10.125]]], "outputs": [[[0.6953125]], [[1.42227395548986e+38]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.10.self_attn.qkv_proj": {"inputs": [[[16.125]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.10.self_attn.o_proj": {"inputs": [[[2.0]]], "outputs": [[[1.6015625]], [[3.09375]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.10.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.89453125]], [[17.25]]]}, "language_model.model.layers.10.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.9375]]]}, "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.89453125]]]}, "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.078125]]]}, "language_model.model.layers.10.self_attn.attn.impl.k_cache": {"inputs": [[[17.25]]]}, "language_model.model.layers.10.self_attn.attn.impl.v_cache": {"inputs": [[[3.1875]]]}, "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.4375]], [[17.25]], [[3.1875]]], "outputs": [[[2.0]], [[1.0]]]}, "language_model.model.layers.10.feed_forward.gate_up_proj": {"inputs": [[[9.875]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.10.feed_forward.down_proj": {"inputs": [[[10.375]]], "outputs": [[[6.90625]], [[11.3125]]], "params": {"weight": [[0.48046875]]}}, "language_model.model.layers.11.self_attn.qkv_proj": {"inputs": [[[13.5625]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.11.self_attn.o_proj": {"inputs": [[[1.9375]]], "outputs": [[[8.0625]], [[7.875]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.11.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.625]], [[8.625]]]}, "language_model.model.layers.11.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.6875]]]}, "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.625]]]}, "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.984375]]]}, "language_model.model.layers.11.self_attn.attn.impl.k_cache": {"inputs": [[[8.625]]]}, "language_model.model.layers.11.self_attn.attn.impl.v_cache": {"inputs": [[[4.6875]]]}, "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[7.59375]], [[8.625]], [[4.6875]]], "outputs": [[[1.9375]], [[1.0]]]}, "language_model.model.layers.11.feed_forward.router": {"inputs": [[[13.375]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op": {"inputs": [[[13.375]]], "outputs": [[[3.90625]], [[20.125]], [[19.0]], [[29.75]], [[7.34375]], [[17.625]], [[28.75]], [[1.2109375]], [[19.875]], [[16.875]], [[8.5625]], [[25.0]], [[26.875]], [[29.75]], [[49.0]], [[26.25]], [[48.25]], [[38.5]], [[44.75]], [[52.5]], [[20.375]], [[16.5]], [[39.75]], [[30.5]], [[28.5]], [[14.125]], [[58.5]], [[21.625]], [[10.4375]], [[11.75]], [[45.0]], [[29.375]], [[9.3125]], [[7.96875]], [[23.0]], [[23.0]], [[22.875]], [[16.25]], [[34.75]], [[38.0]], [[20.125]], [[3.78125]], [[13.875]], [[8.5625]], [[39.25]], [[16.25]], [[2.25]], [[14.8125]], [[35.5]], [[31.25]], [[9.75]], [[33.25]], [[20.25]], [[7.0625]], [[45.0]], [[53.5]], [[38.5]], [[30.75]], [[8.0]], [[34.0]], [[35.25]], [[57.25]], [[10.25]], [[27.0]], [[20.75]], [[17.125]], [[21.75]], [[23.0]], [[46.0]], [[5.5625]], [[51.75]], [[4.90625]], [[4.53125]], [[2.1875]], [[33.0]], [[26.0]], [[17.5]], [[20.875]], [[7.34375]], [[47.0]], [[39.0]], [[19.125]], [[15.3125]], [[13.9375]], [[4.34375]], [[2.265625]], [[1.9140625]], [[33.25]], [[13.25]], [[8.8125]], [[22.5]], [[35.25]], [[10.375]], [[37.0]], [[29.25]], [[15.3125]], [[13.4375]], [[19.375]], [[31.25]], [[11.0]], [[32.5]], [[7.4375]], [[30.875]], [[29.625]], [[17.125]], [[63.0]], [[30.375]], [[9.25]], [[8.0]], [[13.4375]], [[18.5]], [[31.875]], [[9.25]], [[5.0625]], [[13.375]], [[31.625]], [[46.0]], [[26.5]], [[59.0]], [[13.75]], [[29.0]], [[27.75]], [[10.375]], [[14.375]], [[7.46875]], [[9.1875]], [[8.0625]], [[33.25]], [[15.9375]]]}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.392578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[13.375]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.11.feed_forward.shared_expert.down_proj": {"inputs": [[[14.125]]], "outputs": [[[5.625]], [[2.5654100318648876e+38]]], "params": {"weight": [[0.384765625]]}}, "language_model.model.layers.12.self_attn.qkv_proj": {"inputs": [[[18.625]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.12.self_attn.o_proj": {"inputs": [[[3.4375]]], "outputs": [[[1.3359375]], [[3.640625]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.12.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.76953125]], [[11.5625]]]}, "language_model.model.layers.12.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.96875]]]}, "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.76953125]]]}, "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.4375]]]}, "language_model.model.layers.12.self_attn.attn.impl.k_cache": {"inputs": [[[11.5625]]]}, "language_model.model.layers.12.self_attn.attn.impl.v_cache": {"inputs": [[[3.96875]]]}, "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.4375]], [[11.5625]], [[3.96875]]], "outputs": [[[3.1875]], [[1.0]]]}, "language_model.model.layers.12.feed_forward.gate_up_proj": {"inputs": [[[10.75]]], "params": {"weight": [[0.36328125]]}}, "language_model.model.layers.12.feed_forward.down_proj": {"inputs": [[[12.375]]], "outputs": [[[9.0]], [[17.375]]], "params": {"weight": [[0.4453125]]}}, "language_model.model.layers.13.self_attn.qkv_proj": {"inputs": [[[20.875]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.13.self_attn.o_proj": {"inputs": [[[2.0625]]], "outputs": [[[1.6875]], [[6.9375]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.13.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0078125]], [[14.25]]]}, "language_model.model.layers.13.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.96875]]]}, "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0078125]]]}, "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.9375]]]}, "language_model.model.layers.13.self_attn.attn.impl.k_cache": {"inputs": [[[14.25]]]}, "language_model.model.layers.13.self_attn.attn.impl.v_cache": {"inputs": [[[2.96875]]]}, "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.4375]], [[14.25]], [[2.96875]]], "outputs": [[[2.0625]], [[1.0]]]}, "language_model.model.layers.13.feed_forward.router": {"inputs": [[[17.625]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op": {"inputs": [[[17.625]]], "outputs": [[[4.09375]], [[21.0]], [[28.0]], [[22.625]], [[59.0]], [[46.75]], [[8.25]], [[20.25]], [[10.9375]], [[10.375]], [[2.09375]], [[39.5]], [[44.75]], [[30.0]], [[19.375]], [[18.75]], [[19.25]], [[14.8125]], [[26.0]], [[7.8125]], [[25.125]], [[51.5]], [[17.0]], [[8.125]], [[3.46875]], [[15.8125]], [[32.25]], [[41.75]], [[53.25]], [[28.5]], [[6.28125]], [[8.375]], [[17.25]], [[59.25]], [[17.75]], [[33.25]], [[20.625]], [[56.25]], [[51.5]], [[22.0]], [[9.4375]], [[16.0]], [[25.125]], [[22.0]], [[23.625]], [[48.5]], [[59.75]], [[36.75]], [[24.625]], [[19.75]], [[41.75]], [[23.25]], [[40.0]], [[11.8125]], [[56.0]], [[13.8125]], [[29.625]], [[28.125]], [[40.75]], [[39.0]], [[33.0]], [[7.90625]], [[36.0]], [[6.0625]], [[48.75]], [[11.0]], [[52.5]], [[11.6875]], [[28.75]], [[30.75]], [[9.0]], [[54.0]], [[39.75]], [[8.8125]], [[10.75]], [[38.25]], [[24.25]], [[47.75]], [[32.75]], [[25.25]], [[17.125]], [[11.75]], [[21.875]], [[4.0625]], [[35.0]], [[45.75]], [[27.125]], [[21.875]], [[24.875]], [[8.375]], [[49.5]], [[51.25]], [[44.0]], [[9.5625]], [[30.125]], [[57.25]], [[11.0]], [[8.875]], [[10.6875]], [[17.75]], [[58.5]], [[4.5625]], [[1.6015625]], [[28.25]], [[35.25]], [[57.25]], [[28.375]], [[28.5]], [[9.875]], [[16.625]], [[44.5]], [[19.875]], [[28.5]], [[17.875]], [[31.875]], [[25.25]], [[38.5]], [[17.625]], [[46.5]], [[45.25]], [[21.125]], [[42.25]], [[42.25]], [[18.25]], [[11.75]], [[16.625]], [[62.5]], [[25.25]], [[30.625]]]}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.41015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.5078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[17.625]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.13.feed_forward.shared_expert.down_proj": {"inputs": [[[22.375]]], "outputs": [[[4.96875]], [[1.42227395548986e+38]]], "params": {"weight": [[0.6015625]]}}, "language_model.model.layers.14.self_attn.qkv_proj": {"inputs": [[[18.75]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.14.self_attn.o_proj": {"inputs": [[[3.75]]], "outputs": [[[2.140625]], [[4.875]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.14.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.9375]], [[16.25]]]}, "language_model.model.layers.14.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.0625]]]}, "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.9375]]]}, "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.40625]]]}, "language_model.model.layers.14.self_attn.attn.impl.k_cache": {"inputs": [[[16.25]]]}, "language_model.model.layers.14.self_attn.attn.impl.v_cache": {"inputs": [[[5.0625]]]}, "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.25]], [[16.25]], [[5.0625]]], "outputs": [[[3.75]], [[1.0]]]}, "language_model.model.layers.14.feed_forward.gate_up_proj": {"inputs": [[[12.0625]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.14.feed_forward.down_proj": {"inputs": [[[16.875]]], "outputs": [[[4.65625]], [[28.25]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.15.self_attn.qkv_proj": {"inputs": [[[20.875]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.15.self_attn.o_proj": {"inputs": [[[4.125]]], "outputs": [[[21.875]], [[23.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.15.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.78515625]], [[12.4375]]]}, "language_model.model.layers.15.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.28125]]]}, "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.78515625]]]}, "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.125]]]}, "language_model.model.layers.15.self_attn.attn.impl.k_cache": {"inputs": [[[12.4375]]]}, "language_model.model.layers.15.self_attn.attn.impl.v_cache": {"inputs": [[[5.28125]]]}, "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.3125]], [[12.4375]], [[5.28125]]], "outputs": [[[3.8125]], [[1.0]]]}, "language_model.model.layers.15.feed_forward.router": {"inputs": [[[19.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op": {"inputs": [[[19.0]]], "outputs": [[[5.34375]], [[17.0]], [[14.0]], [[26.375]], [[15.75]], [[34.0]], [[36.25]], [[44.75]], [[30.125]], [[59.75]], [[34.5]], [[25.625]], [[16.125]], [[26.375]], [[16.75]], [[3.453125]], [[15.8125]], [[36.5]], [[46.25]], [[21.75]], [[10.9375]], [[46.25]], [[25.0]], [[28.75]], [[49.0]], [[13.1875]], [[7.65625]], [[25.875]], [[36.25]], [[42.75]], [[21.875]], [[43.0]], [[20.625]], [[23.75]], [[16.125]], [[36.0]], [[25.125]], [[15.375]], [[32.0]], [[57.25]], [[49.75]], [[36.0]], [[46.0]], [[45.0]], [[47.25]], [[55.5]], [[34.0]], [[14.125]], [[31.625]], [[12.0625]], [[43.0]], [[20.625]], [[46.5]], [[35.0]], [[27.5]], [[38.0]], [[22.75]], [[37.25]], [[23.125]], [[25.75]], [[43.5]], [[16.5]], [[73.5]], [[48.0]], [[52.0]], [[16.25]], [[57.75]], [[29.125]], [[40.75]], [[13.6875]], [[44.75]], [[36.25]], [[10.5625]], [[21.625]], [[57.25]], [[62.5]], [[30.375]], [[20.125]], [[55.0]], [[29.875]], [[9.3125]], [[34.25]], [[30.25]], [[20.875]], [[12.0]], [[9.0]], [[25.125]], [[22.25]], [[14.0]], [[33.25]], [[28.0]], [[16.125]], [[23.75]], [[24.5]], [[30.375]], [[45.0]], [[36.5]], [[21.875]], [[23.375]], [[13.9375]], [[34.75]], [[30.375]], [[63.25]], [[32.25]], [[23.25]], [[50.0]], [[34.25]], [[9.5]], [[33.5]], [[81.5]], [[16.0]], [[57.0]], [[34.0]], [[37.5]], [[4.25]], [[17.0]], [[8.625]], [[42.75]], [[13.1875]], [[41.5]], [[22.125]], [[63.0]], [[28.75]], [[17.125]], [[29.5]], [[59.25]], [[12.75]], [[11.6875]], [[34.25]]]}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[19.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.15.feed_forward.shared_expert.down_proj": {"inputs": [[[19.0]]], "outputs": [[[4.9375]], [[2.9774707105582116e+38]]], "params": {"weight": [[0.76953125]]}}, "language_model.model.layers.16.self_attn.qkv_proj": {"inputs": [[[20.375]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.16.self_attn.o_proj": {"inputs": [[[3.078125]]], "outputs": [[[2.75]], [[6.28125]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.16.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.98828125]], [[18.75]]]}, "language_model.model.layers.16.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.9375]]]}, "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.98828125]]]}, "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.78125]]]}, "language_model.model.layers.16.self_attn.attn.impl.k_cache": {"inputs": [[[18.75]]]}, "language_model.model.layers.16.self_attn.attn.impl.v_cache": {"inputs": [[[3.9375]]]}, "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.0625]], [[18.75]], [[3.9375]]], "outputs": [[[3.078125]], [[1.0]]]}, "language_model.model.layers.16.feed_forward.gate_up_proj": {"inputs": [[[14.8125]]], "params": {"weight": [[0.341796875]]}}, "language_model.model.layers.16.feed_forward.down_proj": {"inputs": [[[19.625]]], "outputs": [[[7.03125]], [[22.625]]], "params": {"weight": [[0.8359375]]}}, "language_model.model.layers.17.self_attn.qkv_proj": {"inputs": [[[19.875]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.17.self_attn.o_proj": {"inputs": [[[4.90625]]], "outputs": [[[17.5]], [[20.75]]], "params": {"weight": [[0.5859375]]}}, "language_model.model.layers.17.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.90234375]], [[16.125]]]}, "language_model.model.layers.17.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.25]]]}, "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.90234375]]]}, "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.78125]]]}, "language_model.model.layers.17.self_attn.attn.impl.k_cache": {"inputs": [[[16.125]]]}, "language_model.model.layers.17.self_attn.attn.impl.v_cache": {"inputs": [[[5.25]]]}, "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.5625]], [[16.125]], [[5.25]]], "outputs": [[[4.90625]], [[1.0]]]}, "language_model.model.layers.17.feed_forward.router": {"inputs": [[[23.875]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op": {"inputs": [[[23.875]]], "outputs": [[[28.75]], [[37.5]], [[34.25]], [[28.875]], [[7.5625]], [[51.25]], [[26.25]], [[52.25]], [[51.0]], [[58.0]], [[4.46875]], [[12.5625]], [[20.875]], [[37.75]], [[33.75]], [[25.5]], [[49.5]], [[68.5]], [[44.5]], [[26.75]], [[27.375]], [[44.75]], [[35.75]], [[97.5]], [[40.0]], [[6.71875]], [[35.75]], [[15.5625]], [[52.5]], [[66.0]], [[24.75]], [[52.75]], [[22.25]], [[14.25]], [[22.0]], [[26.75]], [[82.0]], [[30.125]], [[17.875]], [[10.5]], [[56.0]], [[29.0]], [[33.25]], [[9.6875]], [[42.5]], [[51.25]], [[41.75]], [[26.375]], [[58.5]], [[34.0]], [[25.375]], [[81.0]], [[49.0]], [[11.375]], [[15.25]], [[38.25]], [[4.28125]], [[8.1875]], [[35.0]], [[12.875]], [[16.75]], [[52.0]], [[15.75]], [[13.625]], [[15.125]], [[37.0]], [[55.5]], [[14.6875]], [[47.25]], [[25.375]], [[27.75]], [[36.25]], [[13.625]], [[40.0]], [[37.0]], [[25.0]], [[18.625]], [[43.0]], [[39.75]], [[10.8125]], [[54.5]], [[20.875]], [[56.0]], [[38.25]], [[35.25]], [[37.5]], [[67.0]], [[18.75]], [[16.25]], [[45.75]], [[55.0]], [[84.5]], [[53.75]], [[7.53125]], [[27.625]], [[18.875]], [[30.25]], [[7.28125]], [[4.21875]], [[33.5]], [[16.75]], [[14.0625]], [[13.5]], [[43.5]], [[32.5]], [[47.5]], [[10.9375]], [[17.625]], [[40.25]], [[63.0]], [[25.25]], [[39.25]], [[26.0]], [[37.25]], [[30.625]], [[32.0]], [[12.0]], [[67.0]], [[26.375]], [[36.75]], [[41.0]], [[27.875]], [[21.25]], [[21.625]], [[29.625]], [[25.625]], [[3.859375]], [[31.375]], [[47.25]]]}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.392578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.3046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.6328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.875]]], "params": {"weight": [[0.388671875]]}}, "language_model.model.layers.17.feed_forward.shared_expert.down_proj": {"inputs": [[[24.0]]], "outputs": [[[2.9375]], [[1.42227395548986e+38]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.18.self_attn.qkv_proj": {"inputs": [[[22.125]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.18.self_attn.o_proj": {"inputs": [[[4.28125]]], "outputs": [[[6.375]], [[11.5625]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.18.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0]], [[16.375]]]}, "language_model.model.layers.18.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.9375]]]}, "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0]]]}, "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.5]]]}, "language_model.model.layers.18.self_attn.attn.impl.k_cache": {"inputs": [[[16.375]]]}, "language_model.model.layers.18.self_attn.attn.impl.v_cache": {"inputs": [[[4.96875]]]}, "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.4375]], [[16.375]], [[4.96875]]], "outputs": [[[4.28125]], [[1.0]]]}, "language_model.model.layers.18.feed_forward.gate_up_proj": {"inputs": [[[18.5]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.18.feed_forward.down_proj": {"inputs": [[[195.0]]], "outputs": [[[8.25]], [[358.0]]], "params": {"weight": [[0.67578125]]}}, "language_model.model.layers.19.self_attn.qkv_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.19.self_attn.o_proj": {"inputs": [[[3.859375]]], "outputs": [[[31.625]], [[36.75]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.19.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.84375]], [[12.375]]]}, "language_model.model.layers.19.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.125]]]}, "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.84375]]]}, "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.609375]]]}, "language_model.model.layers.19.self_attn.attn.impl.k_cache": {"inputs": [[[12.375]]]}, "language_model.model.layers.19.self_attn.attn.impl.v_cache": {"inputs": [[[11.125]]]}, "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.25]], [[12.375]], [[11.125]]], "outputs": [[[3.859375]], [[1.0]]]}, "language_model.model.layers.19.feed_forward.router": {"inputs": [[[23.25]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op": {"inputs": [[[23.25]]], "outputs": [[[5.625]], [[47.5]], [[45.25]], [[10.875]], [[49.0]], [[63.75]], [[18.625]], [[17.875]], [[49.5]], [[13.625]], [[20.125]], [[9.5]], [[22.125]], [[17.0]], [[41.0]], [[44.75]], [[10.8125]], [[8.8125]], [[21.5]], [[32.75]], [[23.0]], [[9.3125]], [[24.875]], [[40.0]], [[29.25]], [[38.75]], [[49.25]], [[23.0]], [[36.25]], [[32.75]], [[37.25]], [[10.25]], [[58.5]], [[40.75]], [[20.5]], [[44.5]], [[38.75]], [[16.5]], [[40.75]], [[41.75]], [[12.3125]], [[53.75]], [[12.375]], [[32.25]], [[31.375]], [[58.0]], [[23.0]], [[16.625]], [[23.0]], [[36.25]], [[61.75]], [[21.0]], [[38.25]], [[17.125]], [[40.0]], [[37.5]], [[12.125]], [[21.0]], [[41.75]], [[48.0]], [[39.5]], [[30.125]], [[67.5]], [[17.75]], [[8.625]], [[54.5]], [[49.25]], [[33.5]], [[22.875]], [[52.75]], [[8.875]], [[33.25]], [[38.25]], [[32.75]], [[30.5]], [[19.0]], [[22.375]], [[22.75]], [[18.25]], [[50.0]], [[13.0625]], [[47.75]], [[19.125]], [[17.5]], [[37.75]], [[21.75]], [[26.625]], [[22.0]], [[16.625]], [[19.25]], [[25.625]], [[23.75]], [[33.25]], [[18.25]], [[39.5]], [[25.625]], [[36.0]], [[15.625]], [[86.0]], [[48.0]], [[10.0625]], [[94.5]], [[23.625]], [[13.25]], [[27.75]], [[40.25]], [[28.25]], [[39.25]], [[19.875]], [[23.25]], [[24.625]], [[44.75]], [[24.5]], [[33.25]], [[32.5]], [[10.3125]], [[8.3125]], [[40.5]], [[32.25]], [[41.75]], [[48.75]], [[10.25]], [[17.875]], [[16.875]], [[34.5]], [[14.75]], [[13.6875]], [[19.875]], [[35.25]]]}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.36328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.43359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.25]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.19.feed_forward.shared_expert.down_proj": {"inputs": [[[27.625]]], "outputs": [[[3.890625]], [[2.9774707105582116e+38]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.20.self_attn.qkv_proj": {"inputs": [[[52.5]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.20.self_attn.o_proj": {"inputs": [[[5.875]]], "outputs": [[[9.5]], [[18.375]]], "params": {"weight": [[0.34765625]]}}, "language_model.model.layers.20.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8359375]], [[18.125]]]}, "language_model.model.layers.20.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.59375]]]}, "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8359375]]]}, "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.3125]]]}, "language_model.model.layers.20.self_attn.attn.impl.k_cache": {"inputs": [[[18.125]]]}, "language_model.model.layers.20.self_attn.attn.impl.v_cache": {"inputs": [[[6.59375]]]}, "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.4375]], [[18.125]], [[6.59375]]], "outputs": [[[5.875]], [[1.0]]]}, "language_model.model.layers.20.feed_forward.gate_up_proj": {"inputs": [[[41.5]]], "params": {"weight": [[0.3671875]]}}, "language_model.model.layers.20.feed_forward.down_proj": {"inputs": [[[16.5]]], "outputs": [[[9.4375]], [[38.5]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.21.self_attn.qkv_proj": {"inputs": [[[56.25]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.21.self_attn.o_proj": {"inputs": [[[3.671875]]], "outputs": [[[25.125]], [[37.75]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.21.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0]], [[16.0]]]}, "language_model.model.layers.21.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.8125]]]}, "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0]]]}, "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.578125]]]}, "language_model.model.layers.21.self_attn.attn.impl.k_cache": {"inputs": [[[16.0]]]}, "language_model.model.layers.21.self_attn.attn.impl.v_cache": {"inputs": [[[8.8125]]]}, "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.125]], [[16.0]], [[8.8125]]], "outputs": [[[3.671875]], [[1.0]]]}, "language_model.model.layers.21.feed_forward.router": {"inputs": [[[24.5]]], "params": {"weight": [[0.45703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op": {"inputs": [[[24.5]]], "outputs": [[[5.0625]], [[38.25]], [[84.5]], [[35.25]], [[53.5]], [[13.3125]], [[89.0]], [[24.5]], [[22.625]], [[30.0]], [[37.25]], [[11.4375]], [[10.6875]], [[13.25]], [[69.5]], [[22.375]], [[50.0]], [[26.125]], [[27.375]], [[25.125]], [[34.25]], [[46.5]], [[34.5]], [[16.125]], [[16.625]], [[25.5]], [[50.5]], [[21.25]], [[13.125]], [[9.125]], [[23.0]], [[7.78125]], [[26.875]], [[42.75]], [[18.625]], [[12.4375]], [[14.25]], [[33.0]], [[16.375]], [[34.75]], [[20.625]], [[7.3125]], [[31.0]], [[23.125]], [[10.3125]], [[39.0]], [[19.125]], [[29.875]], [[56.5]], [[23.25]], [[21.5]], [[82.0]], [[32.5]], [[30.0]], [[22.25]], [[11.1875]], [[29.75]], [[29.875]], [[23.875]], [[35.0]], [[48.75]], [[45.0]], [[17.75]], [[19.875]], [[55.25]], [[14.5625]], [[73.5]], [[13.375]], [[22.75]], [[9.375]], [[15.625]], [[30.875]], [[21.0]], [[23.0]], [[39.0]], [[19.25]], [[10.1875]], [[38.25]], [[7.3125]], [[21.25]], [[24.0]], [[31.0]], [[34.5]], [[50.25]], [[28.75]], [[23.375]], [[36.5]], [[8.0]], [[14.625]], [[9.4375]], [[22.875]], [[36.5]], [[15.3125]], [[63.5]], [[12.6875]], [[12.8125]], [[24.5]], [[28.125]], [[14.0625]], [[37.0]], [[22.75]], [[21.125]], [[25.25]], [[31.625]], [[59.5]], [[36.0]], [[25.5]], [[39.75]], [[46.0]], [[35.75]], [[19.125]], [[70.5]], [[35.0]], [[20.75]], [[20.5]], [[20.125]], [[11.3125]], [[30.25]], [[15.0625]], [[18.5]], [[39.25]], [[15.3125]], [[26.625]], [[35.0]], [[25.875]], [[16.5]], [[41.75]], [[25.5]], [[75.5]]]}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.6015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.396484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.36328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[24.5]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.21.feed_forward.shared_expert.down_proj": {"inputs": [[[17.625]]], "outputs": [[[5.40625]], [[2.5654100318648876e+38]]], "params": {"weight": [[0.486328125]]}}, "language_model.model.layers.22.self_attn.qkv_proj": {"inputs": [[[55.75]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.22.self_attn.o_proj": {"inputs": [[[4.375]]], "outputs": [[[17.25]], [[23.25]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.22.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.125]], [[17.125]]]}, "language_model.model.layers.22.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[5.46875]]]}, "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.125]]]}, "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.46875]]]}, "language_model.model.layers.22.self_attn.attn.impl.k_cache": {"inputs": [[[17.125]]]}, "language_model.model.layers.22.self_attn.attn.impl.v_cache": {"inputs": [[[5.46875]]]}, "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.5]], [[17.125]], [[5.46875]]], "outputs": [[[4.375]], [[1.0]]]}, "language_model.model.layers.22.feed_forward.gate_up_proj": {"inputs": [[[44.75]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.22.feed_forward.down_proj": {"inputs": [[[20.75]]], "outputs": [[[12.0]], [[80.5]]], "params": {"weight": [[0.6796875]]}}, "language_model.model.layers.23.self_attn.qkv_proj": {"inputs": [[[67.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.23.self_attn.o_proj": {"inputs": [[[5.34375]]], "outputs": [[[50.5]], [[61.5]]], "params": {"weight": [[0.32421875]]}}, "language_model.model.layers.23.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.87890625]], [[12.4375]]]}, "language_model.model.layers.23.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.5625]]]}, "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.87890625]]]}, "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.375]]]}, "language_model.model.layers.23.self_attn.attn.impl.k_cache": {"inputs": [[[12.4375]]]}, "language_model.model.layers.23.self_attn.attn.impl.v_cache": {"inputs": [[[8.5625]]]}, "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.5]], [[12.4375]], [[8.5625]]], "outputs": [[[5.34375]], [[1.0]]]}, "language_model.model.layers.23.feed_forward.router": {"inputs": [[[23.625]]], "params": {"weight": [[0.474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op": {"inputs": [[[23.625]]], "outputs": [[[51.25]], [[73.0]], [[7.8125]], [[37.75]], [[45.25]], [[15.875]], [[46.75]], [[21.125]], [[32.5]], [[73.5]], [[25.75]], [[32.5]], [[76.0]], [[55.75]], [[9.375]], [[22.0]], [[47.25]], [[26.0]], [[15.5625]], [[41.5]], [[30.375]], [[18.375]], [[60.25]], [[7.28125]], [[30.875]], [[45.0]], [[15.1875]], [[65.5]], [[60.5]], [[14.3125]], [[12.625]], [[12.125]], [[21.0]], [[19.875]], [[11.5]], [[11.5625]], [[29.5]], [[13.1875]], [[15.625]], [[15.6875]], [[41.0]], [[104.0]], [[20.125]], [[57.5]], [[8.875]], [[24.375]], [[8.5]], [[79.5]], [[36.5]], [[7.5625]], [[57.5]], [[20.875]], [[51.0]], [[31.5]], [[14.875]], [[20.375]], [[23.25]], [[131.0]], [[7.03125]], [[63.0]], [[19.125]], [[20.25]], [[27.125]], [[9.5625]], [[11.0]], [[14.1875]], [[26.125]], [[38.0]], [[47.25]], [[35.75]], [[33.75]], [[5.90625]], [[25.0]], [[34.5]], [[40.75]], [[17.875]], [[51.5]], [[56.75]], [[45.25]], [[31.875]], [[73.5]], [[21.375]], [[23.125]], [[41.25]], [[29.25]], [[10.6875]], [[66.5]], [[18.5]], [[7.3125]], [[24.25]], [[38.0]], [[27.625]], [[41.25]], [[34.5]], [[10.75]], [[17.5]], [[35.0]], [[21.5]], [[29.125]], [[28.625]], [[28.875]], [[15.0]], [[15.9375]], [[12.0625]], [[7.625]], [[33.75]], [[101.0]], [[18.0]], [[27.875]], [[65.5]], [[60.0]], [[9.9375]], [[88.0]], [[7.15625]], [[74.0]], [[29.5]], [[11.9375]], [[53.75]], [[106.0]], [[17.625]], [[10.25]], [[27.0]], [[21.0]], [[34.0]], [[59.0]], [[80.0]], [[20.125]], [[27.125]], [[24.0]]]}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.41796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.39453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.37109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.625]]], "params": {"weight": [[0.43359375]]}}, "language_model.model.layers.23.feed_forward.shared_expert.down_proj": {"inputs": [[[35.0]]], "outputs": [[[24.0]], [[2.5654100318648876e+38]]], "params": {"weight": [[0.9921875]]}}, "language_model.model.layers.24.self_attn.qkv_proj": {"inputs": [[[60.5]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.24.self_attn.o_proj": {"inputs": [[[7.3125]]], "outputs": [[[36.0]], [[42.5]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.24.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.234375]], [[15.5]]]}, "language_model.model.layers.24.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[10.5625]]]}, "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.234375]]]}, "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.46875]]]}, "language_model.model.layers.24.self_attn.attn.impl.k_cache": {"inputs": [[[15.5]]]}, "language_model.model.layers.24.self_attn.attn.impl.v_cache": {"inputs": [[[10.5625]]]}, "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.375]], [[15.5]], [[10.5625]]], "outputs": [[[7.3125]], [[1.0]]]}, "language_model.model.layers.24.feed_forward.gate_up_proj": {"inputs": [[[40.5]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.24.feed_forward.down_proj": {"inputs": [[[33.25]]], "outputs": [[[13.3125]], [[62.5]]], "params": {"weight": [[0.703125]]}}, "language_model.model.layers.25.self_attn.qkv_proj": {"inputs": [[[49.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.25.self_attn.o_proj": {"inputs": [[[7.03125]]], "outputs": [[[47.75]], [[57.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.25.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.046875]], [[17.75]]]}, "language_model.model.layers.25.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.875]]]}, "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.046875]]]}, "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.03125]]]}, "language_model.model.layers.25.self_attn.attn.impl.k_cache": {"inputs": [[[17.75]]]}, "language_model.model.layers.25.self_attn.attn.impl.v_cache": {"inputs": [[[9.875]]]}, "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.5]], [[17.75]], [[9.875]]], "outputs": [[[6.625]], [[1.0]]]}, "language_model.model.layers.25.feed_forward.router": {"inputs": [[[30.375]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op": {"inputs": [[[30.375]]], "outputs": [[[5.65625]], [[85.5]], [[39.5]], [[20.125]], [[14.25]], [[34.0]], [[45.0]], [[14.875]], [[16.5]], [[10.625]], [[15.4375]], [[6.125]], [[64.0]], [[39.75]], [[58.0]], [[69.0]], [[28.25]], [[76.0]], [[13.125]], [[54.75]], [[55.0]], [[37.25]], [[40.75]], [[56.25]], [[31.25]], [[10.1875]], [[13.25]], [[54.5]], [[16.375]], [[9.625]], [[33.25]], [[14.8125]], [[11.1875]], [[9.9375]], [[63.5]], [[34.75]], [[8.75]], [[20.125]], [[19.125]], [[20.875]], [[57.5]], [[25.875]], [[50.5]], [[29.625]], [[43.75]], [[20.375]], [[68.0]], [[89.0]], [[13.1875]], [[15.5]], [[74.0]], [[39.0]], [[11.0625]], [[11.8125]], [[7.84375]], [[33.5]], [[19.125]], [[24.25]], [[15.0]], [[8.1875]], [[18.5]], [[26.75]], [[39.5]], [[15.4375]], [[14.6875]], [[75.0]], [[12.0]], [[43.5]], [[23.5]], [[10.0625]], [[13.3125]], [[14.6875]], [[12.3125]], [[31.625]], [[25.0]], [[65.5]], [[69.0]], [[12.6875]], [[15.3125]], [[25.75]], [[51.5]], [[12.3125]], [[32.5]], [[28.125]], [[45.0]], [[14.125]], [[16.875]], [[26.75]], [[36.0]], [[24.5]], [[48.75]], [[11.3125]], [[18.75]], [[40.5]], [[43.75]], [[8.9375]], [[30.0]], [[31.875]], [[41.0]], [[13.625]], [[39.25]], [[16.625]], [[19.375]], [[27.875]], [[14.75]], [[23.625]], [[21.75]], [[55.75]], [[5.8125]], [[40.75]], [[32.5]], [[34.25]], [[29.375]], [[88.5]], [[104.5]], [[37.0]], [[46.0]], [[25.125]], [[24.5]], [[76.0]], [[40.75]], [[30.625]], [[35.0]], [[26.125]], [[22.375]], [[32.0]], [[7.4375]], [[17.25]], [[10.625]]]}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[30.375]]], "params": {"weight": [[0.419921875]]}}, "language_model.model.layers.25.feed_forward.shared_expert.down_proj": {"inputs": [[[30.75]]], "outputs": [[[29.0]], [[2.5654100318648876e+38]]], "params": {"weight": [[0.8671875]]}}, "language_model.model.layers.26.self_attn.qkv_proj": {"inputs": [[[63.25]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.26.self_attn.o_proj": {"inputs": [[[7.71875]]], "outputs": [[[29.0]], [[53.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.26.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0625]], [[16.875]]]}, "language_model.model.layers.26.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[13.25]]]}, "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0625]]]}, "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.71875]]]}, "language_model.model.layers.26.self_attn.attn.impl.k_cache": {"inputs": [[[16.875]]]}, "language_model.model.layers.26.self_attn.attn.impl.v_cache": {"inputs": [[[13.25]]]}, "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.9375]], [[16.875]], [[13.25]]], "outputs": [[[7.71875]], [[1.0]]]}, "language_model.model.layers.26.feed_forward.gate_up_proj": {"inputs": [[[45.5]]], "params": {"weight": [[0.62890625]]}}, "language_model.model.layers.26.feed_forward.down_proj": {"inputs": [[[20.5]]], "outputs": [[[15.25]], [[103.0]]], "params": {"weight": [[0.87109375]]}}, "language_model.model.layers.27.self_attn.qkv_proj": {"inputs": [[[72.5]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.27.self_attn.o_proj": {"inputs": [[[6.3125]]], "outputs": [[[60.0]], [[66.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.27.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.83984375]], [[15.125]]]}, "language_model.model.layers.27.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.6875]]]}, "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.83984375]]]}, "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.5625]]]}, "language_model.model.layers.27.self_attn.attn.impl.k_cache": {"inputs": [[[15.125]]]}, "language_model.model.layers.27.self_attn.attn.impl.v_cache": {"inputs": [[[11.6875]]]}, "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.5625]], [[15.125]], [[11.6875]]], "outputs": [[[6.3125]], [[1.0]]]}, "language_model.model.layers.27.feed_forward.router": {"inputs": [[[30.875]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op": {"inputs": [[[30.875]]], "outputs": [[[9.125]], [[43.5]], [[42.25]], [[32.25]], [[12.625]], [[11.375]], [[75.5]], [[59.75]], [[31.5]], [[58.5]], [[35.25]], [[32.0]], [[26.125]], [[38.5]], [[24.0]], [[78.5]], [[24.125]], [[22.25]], [[100.0]], [[35.25]], [[20.625]], [[35.25]], [[11.875]], [[19.375]], [[24.5]], [[23.25]], [[16.25]], [[66.0]], [[15.1875]], [[11.9375]], [[35.5]], [[88.0]], [[13.125]], [[61.25]], [[18.0]], [[58.75]], [[24.875]], [[28.75]], [[108.0]], [[55.75]], [[16.375]], [[23.875]], [[42.5]], [[72.0]], [[29.5]], [[21.25]], [[20.5]], [[13.9375]], [[42.75]], [[33.25]], [[13.9375]], [[46.0]], [[10.625]], [[59.75]], [[34.25]], [[48.25]], [[27.625]], [[36.0]], [[15.8125]], [[49.75]], [[10.3125]], [[22.125]], [[47.25]], [[48.0]], [[14.5625]], [[37.25]], [[139.0]], [[14.25]], [[12.5]], [[29.375]], [[9.1875]], [[45.5]], [[33.75]], [[96.0]], [[38.0]], [[28.25]], [[72.5]], [[27.5]], [[57.0]], [[46.0]], [[29.75]], [[46.5]], [[38.0]], [[28.75]], [[48.25]], [[32.5]], [[26.625]], [[7.375]], [[48.5]], [[15.1875]], [[14.375]], [[16.125]], [[72.5]], [[23.5]], [[34.0]], [[43.5]], [[18.875]], [[30.0]], [[53.0]], [[19.5]], [[18.5]], [[99.5]], [[61.25]], [[10.625]], [[26.125]], [[17.25]], [[10.3125]], [[10.25]], [[30.75]], [[17.125]], [[56.25]], [[13.5625]], [[62.5]], [[9.0625]], [[18.625]], [[19.125]], [[20.25]], [[31.0]], [[43.0]], [[36.0]], [[59.75]], [[12.3125]], [[47.25]], [[26.875]], [[102.5]], [[15.375]], [[23.875]], [[15.4375]], [[25.25]]]}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.42578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.341796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[30.875]]], "params": {"weight": [[0.416015625]]}}, "language_model.model.layers.27.feed_forward.shared_expert.down_proj": {"inputs": [[[30.25]]], "outputs": [[[17.375]], [[9.969209968386869e+37]]], "params": {"weight": [[0.60546875]]}}, "language_model.model.layers.28.self_attn.qkv_proj": {"inputs": [[[77.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.28.self_attn.o_proj": {"inputs": [[[7.5]]], "outputs": [[[72.0]], [[74.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.28.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0703125]], [[16.25]]]}, "language_model.model.layers.28.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[10.25]]]}, "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0703125]]]}, "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.03125]]]}, "language_model.model.layers.28.self_attn.attn.impl.k_cache": {"inputs": [[[16.25]]]}, "language_model.model.layers.28.self_attn.attn.impl.v_cache": {"inputs": [[[10.25]]]}, "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.8125]], [[16.25]], [[10.25]]], "outputs": [[[7.5]], [[1.0]]]}, "language_model.model.layers.28.feed_forward.gate_up_proj": {"inputs": [[[40.25]]], "params": {"weight": [[0.3984375]]}}, "language_model.model.layers.28.feed_forward.down_proj": {"inputs": [[[23.75]]], "outputs": [[[95.0]], [[103.5]]], "params": {"weight": [[0.640625]]}}, "language_model.model.layers.29.self_attn.qkv_proj": {"inputs": [[[66.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.29.self_attn.o_proj": {"inputs": [[[6.40625]]], "outputs": [[[70.5]], [[76.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.29.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.9453125]], [[16.125]]]}, "language_model.model.layers.29.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.1875]]]}, "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.9453125]]]}, "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.25]]]}, "language_model.model.layers.29.self_attn.attn.impl.k_cache": {"inputs": [[[16.125]]]}, "language_model.model.layers.29.self_attn.attn.impl.v_cache": {"inputs": [[[8.1875]]]}, "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.75]], [[16.125]], [[8.1875]]], "outputs": [[[6.40625]], [[1.0]]]}, "language_model.model.layers.29.feed_forward.router": {"inputs": [[[31.75]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op": {"inputs": [[[31.75]]], "outputs": [[[8.0]], [[13.75]], [[45.25]], [[46.25]], [[27.125]], [[28.875]], [[35.0]], [[18.875]], [[55.25]], [[54.5]], [[32.5]], [[12.5]], [[12.625]], [[9.75]], [[23.5]], [[34.5]], [[9.0]], [[98.5]], [[40.25]], [[26.375]], [[13.25]], [[28.0]], [[15.5625]], [[26.375]], [[14.1875]], [[91.0]], [[22.875]], [[25.875]], [[16.0]], [[27.25]], [[48.25]], [[92.5]], [[11.3125]], [[23.75]], [[45.0]], [[11.8125]], [[60.75]], [[42.75]], [[26.125]], [[57.25]], [[39.25]], [[15.3125]], [[45.25]], [[13.125]], [[58.75]], [[23.25]], [[30.25]], [[15.875]], [[27.75]], [[52.25]], [[14.0625]], [[14.1875]], [[56.25]], [[72.5]], [[72.5]], [[30.75]], [[34.0]], [[24.875]], [[21.625]], [[39.75]], [[11.6875]], [[43.25]], [[12.5]], [[23.5]], [[30.125]], [[55.5]], [[82.0]], [[21.375]], [[20.875]], [[66.0]], [[20.25]], [[19.25]], [[38.5]], [[15.375]], [[32.75]], [[40.25]], [[42.5]], [[14.6875]], [[19.625]], [[31.5]], [[24.25]], [[23.625]], [[22.375]], [[18.25]], [[46.5]], [[26.75]], [[28.0]], [[21.375]], [[12.0625]], [[61.5]], [[11.9375]], [[26.5]], [[30.375]], [[28.0]], [[38.0]], [[4.09375]], [[20.375]], [[20.25]], [[21.875]], [[36.0]], [[14.75]], [[19.625]], [[39.25]], [[36.0]], [[51.25]], [[9.9375]], [[128.0]], [[70.5]], [[52.5]], [[11.875]], [[21.625]], [[28.0]], [[54.0]], [[61.75]], [[10.4375]], [[26.875]], [[37.75]], [[33.75]], [[46.0]], [[73.5]], [[13.375]], [[28.5]], [[20.25]], [[62.75]], [[52.0]], [[14.8125]], [[25.75]], [[104.0]], [[24.875]]]}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.392578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[31.75]]], "params": {"weight": [[0.3828125]]}}, "language_model.model.layers.29.feed_forward.shared_expert.down_proj": {"inputs": [[[48.5]]], "outputs": [[[8.375]], [[1.42227395548986e+38]]], "params": {"weight": [[0.93359375]]}}, "language_model.model.layers.30.self_attn.qkv_proj": {"inputs": [[[71.5]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.30.self_attn.o_proj": {"inputs": [[[6.28125]]], "outputs": [[[52.25]], [[57.0]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.30.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.5]], [[19.0]]]}, "language_model.model.layers.30.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.0]]]}, "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.5]]]}, "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.46875]]]}, "language_model.model.layers.30.self_attn.attn.impl.k_cache": {"inputs": [[[19.0]]]}, "language_model.model.layers.30.self_attn.attn.impl.v_cache": {"inputs": [[[8.0]]]}, "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[16.875]], [[19.0]], [[8.0]]], "outputs": [[[6.28125]], [[1.0]]]}, "language_model.model.layers.30.feed_forward.gate_up_proj": {"inputs": [[[32.0]]], "params": {"weight": [[0.37109375]]}}, "language_model.model.layers.30.feed_forward.down_proj": {"inputs": [[[34.5]]], "outputs": [[[18.625]], [[139.0]]], "params": {"weight": [[0.58984375]]}}, "language_model.model.layers.31.self_attn.qkv_proj": {"inputs": [[[71.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.self_attn.o_proj": {"inputs": [[[7.03125]]], "outputs": [[[66.5]], [[88.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.31.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0]], [[13.4375]]]}, "language_model.model.layers.31.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.5]]]}, "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0]]]}, "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.40625]]]}, "language_model.model.layers.31.self_attn.attn.impl.k_cache": {"inputs": [[[13.4375]]]}, "language_model.model.layers.31.self_attn.attn.impl.v_cache": {"inputs": [[[9.5]]]}, "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.3125]], [[13.4375]], [[9.5]]], "outputs": [[[7.03125]], [[1.0]]]}, "language_model.model.layers.31.feed_forward.router": {"inputs": [[[33.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op": {"inputs": [[[33.0]]], "outputs": [[[13.0]], [[14.25]], [[100.0]], [[25.875]], [[28.125]], [[72.5]], [[43.25]], [[58.0]], [[12.75]], [[33.75]], [[52.5]], [[54.75]], [[14.0625]], [[17.375]], [[14.8125]], [[24.25]], [[14.625]], [[40.0]], [[26.375]], [[75.0]], [[18.5]], [[8.0625]], [[19.75]], [[32.75]], [[75.0]], [[67.5]], [[39.0]], [[41.0]], [[41.0]], [[21.5]], [[14.6875]], [[21.125]], [[21.375]], [[43.25]], [[72.5]], [[24.875]], [[11.8125]], [[32.75]], [[28.5]], [[36.5]], [[62.25]], [[62.25]], [[40.5]], [[14.625]], [[35.5]], [[62.5]], [[47.75]], [[40.0]], [[21.375]], [[21.625]], [[48.0]], [[45.25]], [[56.25]], [[72.0]], [[19.5]], [[67.0]], [[27.625]], [[13.5]], [[55.0]], [[26.125]], [[21.375]], [[32.75]], [[41.0]], [[38.0]], [[57.0]], [[61.0]], [[80.5]], [[21.0]], [[98.5]], [[33.75]], [[20.5]], [[45.0]], [[46.5]], [[26.0]], [[39.75]], [[12.875]], [[13.125]], [[38.75]], [[62.25]], [[37.75]], [[52.0]], [[23.75]], [[80.5]], [[22.5]], [[72.5]], [[16.0]], [[17.375]], [[64.5]], [[22.375]], [[36.25]], [[62.5]], [[93.0]], [[29.125]], [[31.75]], [[13.3125]], [[47.0]], [[24.875]], [[48.5]], [[19.25]], [[20.375]], [[24.0]], [[21.5]], [[8.375]], [[32.25]], [[27.625]], [[31.125]], [[7.9375]], [[34.5]], [[12.3125]], [[24.0]], [[57.25]], [[47.5]], [[47.25]], [[28.875]], [[20.5]], [[20.25]], [[47.0]], [[60.75]], [[14.0]], [[24.125]], [[52.75]], [[31.375]], [[26.75]], [[13.0]], [[13.5625]], [[24.625]], [[16.25]], [[18.5]], [[37.25]]]}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.380859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.45703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.365234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.357421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[33.0]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.31.feed_forward.shared_expert.down_proj": {"inputs": [[[118.5]]], "outputs": [[[84.0]], [[2.5654100318648876e+38]]], "params": {"weight": [[0.75]]}}, "language_model.model.layers.32.self_attn.qkv_proj": {"inputs": [[[53.25]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.32.self_attn.o_proj": {"inputs": [[[6.0]]], "outputs": [[[96.0]], [[105.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.32.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.4453125]], [[16.125]]]}, "language_model.model.layers.32.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.5]]]}, "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.4453125]]]}, "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.5]]]}, "language_model.model.layers.32.self_attn.attn.impl.k_cache": {"inputs": [[[16.125]]]}, "language_model.model.layers.32.self_attn.attn.impl.v_cache": {"inputs": [[[7.5]]]}, "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[16.5]], [[16.125]], [[7.5]]], "outputs": [[[6.0]], [[1.0]]]}, "language_model.model.layers.32.feed_forward.gate_up_proj": {"inputs": [[[32.75]]], "params": {"weight": [[0.400390625]]}}, "language_model.model.layers.32.feed_forward.down_proj": {"inputs": [[[56.75]]], "outputs": [[[21.125]], [[79.5]]], "params": {"weight": [[0.474609375]]}}, "language_model.model.layers.33.self_attn.qkv_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.33.self_attn.o_proj": {"inputs": [[[7.25]]], "outputs": [[[97.0]], [[118.0]]], "params": {"weight": [[0.384765625]]}}, "language_model.model.layers.33.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.234375]], [[17.75]]]}, "language_model.model.layers.33.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.125]]]}, "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.234375]]]}, "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.5625]]]}, "language_model.model.layers.33.self_attn.attn.impl.k_cache": {"inputs": [[[17.75]]]}, "language_model.model.layers.33.self_attn.attn.impl.v_cache": {"inputs": [[[9.125]]]}, "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.5]], [[17.75]], [[9.125]]], "outputs": [[[6.8125]], [[1.0]]]}, "language_model.model.layers.33.feed_forward.router": {"inputs": [[[64.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op": {"inputs": [[[64.0]]], "outputs": [[[9.6875]], [[55.25]], [[42.5]], [[41.5]], [[31.0]], [[21.375]], [[69.5]], [[18.625]], [[46.75]], [[36.75]], [[46.5]], [[30.125]], [[43.5]], [[40.25]], [[27.0]], [[52.25]], [[42.25]], [[24.0]], [[34.75]], [[17.25]], [[17.75]], [[18.375]], [[20.0]], [[27.875]], [[30.875]], [[31.25]], [[35.5]], [[88.0]], [[54.5]], [[49.75]], [[41.0]], [[30.5]], [[46.25]], [[17.75]], [[80.5]], [[27.125]], [[22.75]], [[29.625]], [[15.625]], [[84.5]], [[36.5]], [[16.0]], [[20.5]], [[61.75]], [[68.5]], [[39.25]], [[20.125]], [[92.0]], [[21.0]], [[57.5]], [[114.0]], [[42.5]], [[16.125]], [[49.0]], [[24.375]], [[131.0]], [[138.0]], [[49.0]], [[21.5]], [[32.75]], [[69.5]], [[47.75]], [[30.75]], [[26.625]], [[109.5]], [[20.875]], [[123.5]], [[18.625]], [[17.125]], [[110.5]], [[37.0]], [[36.75]], [[66.5]], [[79.5]], [[65.5]], [[36.5]], [[67.0]], [[44.25]], [[31.0]], [[12.1875]], [[48.0]], [[28.0]], [[9.25]], [[88.0]], [[25.5]], [[16.5]], [[66.5]], [[60.75]], [[17.5]], [[89.0]], [[43.0]], [[19.25]], [[18.5]], [[24.25]], [[40.5]], [[51.0]], [[91.5]], [[29.75]], [[77.0]], [[38.25]], [[83.0]], [[106.5]], [[53.5]], [[28.125]], [[41.75]], [[67.5]], [[15.6875]], [[81.0]], [[29.5]], [[22.125]], [[50.75]], [[52.5]], [[153.0]], [[12.25]], [[21.125]], [[76.5]], [[31.0]], [[22.625]], [[39.5]], [[29.0]], [[65.0]], [[30.0]], [[35.5]], [[18.0]], [[9.0]], [[48.5]], [[37.0]], [[14.9375]], [[17.5]]]}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.455078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.3046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[64.0]]], "params": {"weight": [[0.431640625]]}}, "language_model.model.layers.33.feed_forward.shared_expert.down_proj": {"inputs": [[[61.75]]], "outputs": [[[53.75]], [[1.42227395548986e+38]]], "params": {"weight": [[0.7109375]]}}, "language_model.model.layers.34.self_attn.qkv_proj": {"inputs": [[[54.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.34.self_attn.o_proj": {"inputs": [[[7.0]]], "outputs": [[[46.75]], [[80.5]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.34.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.2109375]], [[18.75]]]}, "language_model.model.layers.34.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.8125]]]}, "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.2109375]]]}, "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.8125]]]}, "language_model.model.layers.34.self_attn.attn.impl.k_cache": {"inputs": [[[18.75]]]}, "language_model.model.layers.34.self_attn.attn.impl.v_cache": {"inputs": [[[8.8125]]]}, "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.3125]], [[18.75]], [[8.8125]]], "outputs": [[[7.0]], [[1.0]]]}, "language_model.model.layers.34.feed_forward.gate_up_proj": {"inputs": [[[31.125]]], "params": {"weight": [[0.50390625]]}}, "language_model.model.layers.34.feed_forward.down_proj": {"inputs": [[[86.5]]], "outputs": [[[30.375]], [[128.0]]], "params": {"weight": [[0.6640625]]}}, "language_model.model.layers.35.self_attn.qkv_proj": {"inputs": [[[52.75]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.35.self_attn.o_proj": {"inputs": [[[6.59375]]], "outputs": [[[87.5]], [[93.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.35.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.078125]], [[16.25]]]}, "language_model.model.layers.35.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.4375]]]}, "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.078125]]]}, "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.3125]]]}, "language_model.model.layers.35.self_attn.attn.impl.k_cache": {"inputs": [[[16.25]]]}, "language_model.model.layers.35.self_attn.attn.impl.v_cache": {"inputs": [[[9.4375]]]}, "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.625]], [[16.25]], [[9.4375]]], "outputs": [[[6.59375]], [[1.0]]]}, "language_model.model.layers.35.feed_forward.router": {"inputs": [[[64.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op": {"inputs": [[[64.0]]], "outputs": [[[13.3125]], [[183.0]], [[41.5]], [[15.5]], [[48.25]], [[107.0]], [[35.25]], [[61.5]], [[91.0]], [[85.5]], [[51.0]], [[78.5]], [[47.25]], [[62.75]], [[56.5]], [[71.0]], [[48.25]], [[59.75]], [[38.25]], [[227.0]], [[57.75]], [[65.0]], [[33.75]], [[27.875]], [[47.75]], [[34.5]], [[60.25]], [[87.0]], [[49.25]], [[27.875]], [[78.0]], [[74.5]], [[42.5]], [[49.25]], [[45.25]], [[56.25]], [[70.0]], [[44.25]], [[51.5]], [[49.0]], [[47.75]], [[54.5]], [[51.25]], [[26.25]], [[34.5]], [[39.25]], [[40.5]], [[18.125]], [[29.625]], [[67.0]], [[95.5]], [[36.75]], [[44.25]], [[54.0]], [[47.25]], [[68.0]], [[30.125]], [[75.0]], [[37.0]], [[46.75]], [[70.0]], [[77.0]], [[19.25]], [[63.0]], [[44.0]], [[38.0]], [[22.0]], [[117.0]], [[24.375]], [[20.875]], [[107.0]], [[36.5]], [[148.0]], [[33.75]], [[39.5]], [[47.75]], [[42.5]], [[36.5]], [[57.25]], [[84.0]], [[110.0]], [[83.0]], [[142.0]], [[40.0]], [[35.25]], [[106.0]], [[126.5]], [[42.75]], [[26.875]], [[20.625]], [[44.25]], [[81.5]], [[45.0]], [[75.5]], [[108.5]], [[95.0]], [[74.0]], [[74.0]], [[50.75]], [[101.0]], [[41.0]], [[136.0]], [[26.875]], [[39.75]], [[52.0]], [[15.625]], [[53.5]], [[96.0]], [[25.25]], [[100.5]], [[48.75]], [[60.25]], [[30.625]], [[83.0]], [[65.5]], [[28.875]], [[59.75]], [[65.0]], [[18.625]], [[54.25]], [[58.5]], [[87.0]], [[18.375]], [[23.125]], [[100.5]], [[96.0]], [[56.75]], [[20.375]], [[31.125]]]}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.435546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[64.0]]], "params": {"weight": [[0.34765625]]}}, "language_model.model.layers.35.feed_forward.shared_expert.down_proj": {"inputs": [[[93.0]]], "outputs": [[[101.0]], [[1.42227395548986e+38]]], "params": {"weight": [[0.90234375]]}}, "language_model.model.layers.36.self_attn.qkv_proj": {"inputs": [[[56.75]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.36.self_attn.o_proj": {"inputs": [[[8.4375]]], "outputs": [[[27.75]], [[43.0]]], "params": {"weight": [[0.396484375]]}}, "language_model.model.layers.36.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.140625]], [[19.125]]]}, "language_model.model.layers.36.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.4375]]]}, "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.140625]]]}, "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.15625]]]}, "language_model.model.layers.36.self_attn.attn.impl.k_cache": {"inputs": [[[19.125]]]}, "language_model.model.layers.36.self_attn.attn.impl.v_cache": {"inputs": [[[9.4375]]]}, "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.1875]], [[19.125]], [[9.4375]]], "outputs": [[[8.4375]], [[1.0]]]}, "language_model.model.layers.36.feed_forward.gate_up_proj": {"inputs": [[[29.625]]], "params": {"weight": [[0.72265625]]}}, "language_model.model.layers.36.feed_forward.down_proj": {"inputs": [[[67.0]]], "outputs": [[[38.25]], [[76.5]]], "params": {"weight": [[0.859375]]}}, "language_model.model.layers.37.self_attn.qkv_proj": {"inputs": [[[50.25]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.37.self_attn.o_proj": {"inputs": [[[10.1875]]], "outputs": [[[102.0]], [[105.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.37.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.8125]], [[18.75]]]}, "language_model.model.layers.37.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[13.5]]]}, "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.8125]]]}, "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[8.9375]]]}, "language_model.model.layers.37.self_attn.attn.impl.k_cache": {"inputs": [[[18.75]]]}, "language_model.model.layers.37.self_attn.attn.impl.v_cache": {"inputs": [[[13.5]]]}, "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[22.0]], [[18.75]], [[13.5]]], "outputs": [[[10.1875]], [[1.0]]]}, "language_model.model.layers.37.feed_forward.router": {"inputs": [[[70.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op": {"inputs": [[[70.0]]], "outputs": [[[11.0625]], [[103.0]], [[70.0]], [[44.25]], [[54.0]], [[81.0]], [[32.75]], [[54.75]], [[95.0]], [[28.25]], [[84.5]], [[26.0]], [[20.375]], [[63.0]], [[40.0]], [[78.0]], [[39.0]], [[41.25]], [[62.75]], [[106.5]], [[40.75]], [[66.5]], [[47.75]], [[77.0]], [[52.75]], [[87.0]], [[53.25]], [[63.5]], [[64.5]], [[161.0]], [[52.5]], [[23.5]], [[129.0]], [[60.0]], [[28.125]], [[76.5]], [[34.25]], [[97.5]], [[119.0]], [[102.0]], [[69.5]], [[57.75]], [[34.75]], [[42.75]], [[41.0]], [[38.0]], [[102.5]], [[105.0]], [[25.625]], [[61.75]], [[20.0]], [[42.5]], [[57.5]], [[82.0]], [[56.0]], [[54.75]], [[70.0]], [[92.5]], [[45.5]], [[82.5]], [[117.5]], [[91.5]], [[43.5]], [[73.0]], [[46.75]], [[74.5]], [[31.25]], [[34.75]], [[92.0]], [[99.0]], [[29.625]], [[49.25]], [[51.5]], [[34.0]], [[39.75]], [[31.125]], [[66.0]], [[158.0]], [[20.25]], [[64.5]], [[140.0]], [[70.5]], [[27.875]], [[110.5]], [[32.0]], [[44.5]], [[40.75]], [[78.0]], [[82.5]], [[86.5]], [[178.0]], [[33.75]], [[60.75]], [[41.0]], [[50.75]], [[32.25]], [[70.5]], [[23.25]], [[25.625]], [[119.5]], [[63.25]], [[148.0]], [[49.0]], [[75.5]], [[39.75]], [[90.5]], [[73.5]], [[105.5]], [[24.0]], [[61.75]], [[40.5]], [[43.0]], [[61.25]], [[47.25]], [[121.5]], [[42.25]], [[75.0]], [[61.75]], [[85.5]], [[37.25]], [[67.0]], [[30.75]], [[33.0]], [[69.5]], [[77.0]], [[96.0]], [[30.125]], [[49.0]], [[64.5]]]}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.361328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.42578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[70.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.37.feed_forward.shared_expert.down_proj": {"inputs": [[[90.5]]], "outputs": [[[15.875]], [[2.5654100318648876e+38]]], "params": {"weight": [[0.4453125]]}}, "language_model.model.layers.38.self_attn.qkv_proj": {"inputs": [[[47.25]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.38.self_attn.o_proj": {"inputs": [[[8.0]]], "outputs": [[[26.75]], [[56.0]]], "params": {"weight": [[0.91796875]]}}, "language_model.model.layers.38.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.328125]], [[13.5]]]}, "language_model.model.layers.38.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.5]]]}, "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.328125]]]}, "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.71875]]]}, "language_model.model.layers.38.self_attn.attn.impl.k_cache": {"inputs": [[[13.5]]]}, "language_model.model.layers.38.self_attn.attn.impl.v_cache": {"inputs": [[[9.5]]]}, "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[17.875]], [[13.5]], [[9.5]]], "outputs": [[[8.0]], [[1.0]]]}, "language_model.model.layers.38.feed_forward.gate_up_proj": {"inputs": [[[31.125]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.38.feed_forward.down_proj": {"inputs": [[[118.0]]], "outputs": [[[24.75]], [[193.0]]], "params": {"weight": [[0.67578125]]}}, "language_model.model.layers.39.self_attn.qkv_proj": {"inputs": [[[49.75]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.39.self_attn.o_proj": {"inputs": [[[5.5625]]], "outputs": [[[129.0]], [[134.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.39.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0234375]], [[8.9375]]]}, "language_model.model.layers.39.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[15.0]]]}, "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0234375]]]}, "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[9.375]]]}, "language_model.model.layers.39.self_attn.attn.impl.k_cache": {"inputs": [[[8.9375]]]}, "language_model.model.layers.39.self_attn.attn.impl.v_cache": {"inputs": [[[15.0]]]}, "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.875]], [[8.9375]], [[15.0]]], "outputs": [[[5.5625]], [[1.0]]]}, "language_model.model.layers.39.feed_forward.router": {"inputs": [[[61.5]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op": {"inputs": [[[61.5]]], "outputs": [[[11.125]], [[60.0]], [[39.25]], [[79.5]], [[80.5]], [[46.75]], [[54.75]], [[44.5]], [[66.5]], [[50.25]], [[133.0]], [[67.0]], [[25.75]], [[112.0]], [[103.0]], [[68.0]], [[50.0]], [[103.0]], [[94.5]], [[57.25]], [[54.0]], [[43.75]], [[99.0]], [[40.5]], [[121.5]], [[45.0]], [[97.5]], [[72.5]], [[133.0]], [[67.0]], [[37.75]], [[60.0]], [[140.0]], [[29.625]], [[52.0]], [[130.0]], [[59.0]], [[94.0]], [[77.0]], [[97.5]], [[67.5]], [[84.0]], [[111.0]], [[70.5]], [[37.25]], [[62.75]], [[28.625]], [[59.75]], [[49.75]], [[90.0]], [[56.0]], [[80.5]], [[67.0]], [[48.75]], [[37.25]], [[44.75]], [[78.5]], [[37.5]], [[115.0]], [[39.0]], [[34.0]], [[144.0]], [[82.0]], [[116.0]], [[15.4375]], [[42.0]], [[40.25]], [[54.0]], [[108.5]], [[94.0]], [[118.0]], [[96.5]], [[44.0]], [[82.5]], [[108.0]], [[55.25]], [[65.0]], [[65.0]], [[107.0]], [[98.0]], [[59.5]], [[52.75]], [[123.0]], [[109.5]], [[67.0]], [[109.5]], [[114.5]], [[134.0]], [[78.5]], [[88.5]], [[70.0]], [[64.0]], [[99.0]], [[55.0]], [[58.0]], [[47.5]], [[61.5]], [[54.0]], [[73.5]], [[79.5]], [[44.0]], [[71.5]], [[52.25]], [[86.0]], [[96.5]], [[74.0]], [[112.0]], [[89.0]], [[139.0]], [[64.5]], [[84.0]], [[51.75]], [[69.0]], [[38.25]], [[88.0]], [[52.75]], [[111.0]], [[24.0]], [[39.0]], [[85.0]], [[75.0]], [[28.75]], [[70.0]], [[105.5]], [[40.0]], [[81.0]], [[99.5]], [[69.0]], [[83.5]]]}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[61.5]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.39.feed_forward.shared_expert.down_proj": {"inputs": [[[139.0]]], "outputs": [[[10.8125]], [[2.9907629905160607e+38]]], "params": {"weight": [[0.388671875]]}}, "language_model.model.layers.40.self_attn.qkv_proj": {"inputs": [[[46.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.40.self_attn.o_proj": {"inputs": [[[9.8125]]], "outputs": [[[93.0]], [[90.0]]], "params": {"weight": [[0.365234375]]}}, "language_model.model.layers.40.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.0859375]], [[15.5]]]}, "language_model.model.layers.40.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.25]]]}, "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.0859375]]]}, "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[9.8125]]]}, "language_model.model.layers.40.self_attn.attn.impl.k_cache": {"inputs": [[[15.5]]]}, "language_model.model.layers.40.self_attn.attn.impl.v_cache": {"inputs": [[[11.25]]]}, "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.875]], [[15.5]], [[11.25]]], "outputs": [[[9.4375]], [[1.0]]]}, "language_model.model.layers.40.feed_forward.gate_up_proj": {"inputs": [[[40.75]]], "params": {"weight": [[0.3046875]]}}, "language_model.model.layers.40.feed_forward.down_proj": {"inputs": [[[148.0]]], "outputs": [[[22.625]], [[146.0]]], "params": {"weight": [[0.451171875]]}}, "language_model.model.layers.41.self_attn.qkv_proj": {"inputs": [[[33.75]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.41.self_attn.o_proj": {"inputs": [[[7.3125]]], "outputs": [[[159.0]], [[164.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.41.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.4921875]], [[15.0625]]]}, "language_model.model.layers.41.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.0]]]}, "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.4921875]]]}, "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.0]]]}, "language_model.model.layers.41.self_attn.attn.impl.k_cache": {"inputs": [[[15.0625]]]}, "language_model.model.layers.41.self_attn.attn.impl.v_cache": {"inputs": [[[9.0]]]}, "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[19.0]], [[15.0625]], [[9.0]]], "outputs": [[[7.3125]], [[1.0]]]}, "language_model.model.layers.41.feed_forward.router": {"inputs": [[[68.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op": {"inputs": [[[68.0]]], "outputs": [[[25.375]], [[60.5]], [[35.25]], [[37.5]], [[86.5]], [[42.25]], [[56.75]], [[111.5]], [[51.0]], [[50.25]], [[105.5]], [[110.0]], [[64.0]], [[85.0]], [[78.0]], [[143.0]], [[139.0]], [[90.0]], [[104.0]], [[123.5]], [[50.0]], [[89.0]], [[43.0]], [[72.0]], [[141.0]], [[139.0]], [[191.0]], [[145.0]], [[44.75]], [[51.5]], [[58.5]], [[68.0]], [[81.0]], [[193.0]], [[91.5]], [[86.0]], [[135.0]], [[44.25]], [[116.0]], [[45.25]], [[41.5]], [[42.0]], [[153.0]], [[55.5]], [[42.25]], [[42.5]], [[101.0]], [[100.5]], [[57.75]], [[185.0]], [[148.0]], [[171.0]], [[83.5]], [[161.0]], [[89.5]], [[104.5]], [[65.5]], [[134.0]], [[71.5]], [[81.5]], [[52.5]], [[119.5]], [[132.0]], [[60.0]], [[62.0]], [[78.0]], [[53.5]], [[197.0]], [[54.0]], [[44.0]], [[139.0]], [[39.5]], [[58.75]], [[98.5]], [[24.875]], [[125.5]], [[76.5]], [[98.5]], [[47.25]], [[57.0]], [[86.0]], [[144.0]], [[127.0]], [[153.0]], [[73.0]], [[49.75]], [[40.5]], [[76.0]], [[40.5]], [[67.5]], [[121.0]], [[62.75]], [[114.5]], [[95.0]], [[120.5]], [[92.5]], [[78.0]], [[46.75]], [[63.5]], [[88.0]], [[114.0]], [[136.0]], [[67.0]], [[80.5]], [[46.5]], [[75.5]], [[60.0]], [[92.0]], [[77.0]], [[32.5]], [[102.5]], [[164.0]], [[130.0]], [[117.5]], [[89.0]], [[85.5]], [[166.0]], [[71.5]], [[127.0]], [[144.0]], [[94.5]], [[105.0]], [[70.0]], [[89.0]], [[49.5]], [[53.25]], [[58.0]], [[129.0]], [[126.0]]]}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.32421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.32421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.361328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.498046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[68.0]]], "params": {"weight": [[0.421875]]}}, "language_model.model.layers.41.feed_forward.shared_expert.down_proj": {"inputs": [[[262.0]]], "outputs": [[[35.25]], [[1.42227395548986e+38]]], "params": {"weight": [[0.4609375]]}}, "language_model.model.layers.42.self_attn.qkv_proj": {"inputs": [[[38.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.42.self_attn.o_proj": {"inputs": [[[9.375]]], "outputs": [[[50.5]], [[134.0]]], "params": {"weight": [[0.87109375]]}}, "language_model.model.layers.42.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.6015625]], [[15.6875]]]}, "language_model.model.layers.42.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.0625]]]}, "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.6015625]]]}, "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[9.0]]]}, "language_model.model.layers.42.self_attn.attn.impl.k_cache": {"inputs": [[[15.6875]]]}, "language_model.model.layers.42.self_attn.attn.impl.v_cache": {"inputs": [[[11.0625]]]}, "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[20.0]], [[15.6875]], [[11.0625]]], "outputs": [[[9.375]], [[1.0]]]}, "language_model.model.layers.42.feed_forward.gate_up_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.42.feed_forward.down_proj": {"inputs": [[[184.0]]], "outputs": [[[59.5]], [[338.0]]], "params": {"weight": [[0.765625]]}}, "language_model.model.layers.43.self_attn.qkv_proj": {"inputs": [[[45.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.43.self_attn.o_proj": {"inputs": [[[9.3125]]], "outputs": [[[94.5]], [[232.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.43.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.015625]], [[14.0]]]}, "language_model.model.layers.43.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[10.75]]]}, "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.015625]]]}, "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[8.5]]]}, "language_model.model.layers.43.self_attn.attn.impl.k_cache": {"inputs": [[[14.0]]]}, "language_model.model.layers.43.self_attn.attn.impl.v_cache": {"inputs": [[[10.75]]]}, "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.4375]], [[14.0]], [[10.75]]], "outputs": [[[9.3125]], [[1.0]]]}, "language_model.model.layers.43.feed_forward.router": {"inputs": [[[68.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op": {"inputs": [[[68.0]]], "outputs": [[[22.0]], [[53.25]], [[85.0]], [[64.0]], [[60.75]], [[179.0]], [[40.25]], [[54.5]], [[141.0]], [[119.5]], [[112.5]], [[108.0]], [[118.0]], [[74.5]], [[91.0]], [[64.5]], [[82.5]], [[138.0]], [[88.0]], [[206.0]], [[47.0]], [[178.0]], [[125.5]], [[160.0]], [[110.5]], [[116.5]], [[89.0]], [[75.5]], [[59.5]], [[86.0]], [[170.0]], [[53.25]], [[74.5]], [[63.75]], [[128.0]], [[73.0]], [[137.0]], [[73.5]], [[97.5]], [[71.0]], [[127.0]], [[94.0]], [[125.5]], [[157.0]], [[46.5]], [[217.0]], [[110.5]], [[114.0]], [[36.5]], [[126.0]], [[114.5]], [[49.25]], [[152.0]], [[149.0]], [[164.0]], [[69.0]], [[55.0]], [[39.0]], [[39.0]], [[77.5]], [[105.0]], [[50.75]], [[67.5]], [[50.5]], [[163.0]], [[118.0]], [[98.5]], [[71.0]], [[62.25]], [[189.0]], [[51.75]], [[91.0]], [[56.25]], [[60.75]], [[70.0]], [[40.0]], [[102.5]], [[152.0]], [[56.5]], [[85.5]], [[85.5]], [[97.0]], [[94.5]], [[132.0]], [[78.0]], [[81.0]], [[110.0]], [[37.25]], [[106.0]], [[77.5]], [[70.0]], [[114.0]], [[67.0]], [[100.5]], [[156.0]], [[71.5]], [[52.25]], [[48.5]], [[114.0]], [[55.25]], [[73.0]], [[66.5]], [[86.0]], [[72.0]], [[77.5]], [[75.5]], [[100.0]], [[79.0]], [[79.5]], [[57.0]], [[84.5]], [[119.0]], [[142.0]], [[111.5]], [[57.75]], [[38.75]], [[171.0]], [[118.0]], [[127.5]], [[118.5]], [[74.0]], [[60.0]], [[136.0]], [[105.0]], [[135.0]], [[148.0]], [[159.0]], [[81.0]], [[51.25]]]}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.3046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.455078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.4609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[68.0]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.43.feed_forward.shared_expert.down_proj": {"inputs": [[[233.0]]], "outputs": [[[23.25]], [[1.42227395548986e+38]]], "params": {"weight": [[0.439453125]]}}, "language_model.model.layers.44.self_attn.qkv_proj": {"inputs": [[[28.625]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.44.self_attn.o_proj": {"inputs": [[[15.625]]], "outputs": [[[68.0]], [[153.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.44.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.140625]], [[17.25]]]}, "language_model.model.layers.44.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[21.125]]]}, "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.140625]]]}, "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[15.625]]]}, "language_model.model.layers.44.self_attn.attn.impl.k_cache": {"inputs": [[[17.25]]]}, "language_model.model.layers.44.self_attn.attn.impl.v_cache": {"inputs": [[[21.125]]]}, "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.8125]], [[17.25]], [[21.125]]], "outputs": [[[14.75]], [[1.0]]]}, "language_model.model.layers.44.feed_forward.gate_up_proj": {"inputs": [[[41.5]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.44.feed_forward.down_proj": {"inputs": [[[212.0]]], "outputs": [[[63.25]], [[191.0]]], "params": {"weight": [[0.640625]]}}, "language_model.model.layers.45.self_attn.qkv_proj": {"inputs": [[[27.375]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.45.self_attn.o_proj": {"inputs": [[[13.5625]]], "outputs": [[[169.0]], [[220.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.45.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.5]], [[18.125]]]}, "language_model.model.layers.45.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[19.25]]]}, "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.5]]]}, "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[13.5625]]]}, "language_model.model.layers.45.self_attn.attn.impl.k_cache": {"inputs": [[[18.625]]]}, "language_model.model.layers.45.self_attn.attn.impl.v_cache": {"inputs": [[[19.25]]]}, "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[19.0]], [[18.625]], [[19.25]]], "outputs": [[[12.5]], [[1.0]]]}, "language_model.model.layers.45.feed_forward.router": {"inputs": [[[51.75]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op": {"inputs": [[[51.75]]], "outputs": [[[10.625]], [[56.5]], [[30.125]], [[22.75]], [[66.5]], [[0.0]], [[52.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[41.5]], [[41.25]], [[31.0]], [[56.0]], [[0.0]], [[0.0]], [[0.0]], [[38.5]], [[0.0]], [[0.0]], [[0.0]], [[47.0]], [[0.0]], [[30.625]], [[40.75]], [[16.125]], [[72.0]], [[64.0]], [[0.0]], [[31.875]], [[22.75]], [[37.25]], [[43.0]], [[49.75]], [[27.625]], [[0.0]], [[26.5]], [[21.125]], [[100.5]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[67.0]], [[0.0]], [[66.5]], [[0.0]], [[0.0]], [[0.0]], [[53.75]], [[0.0]], [[82.5]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[42.5]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[59.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[78.0]], [[72.5]], [[63.25]], [[24.25]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[75.0]], [[0.0]], [[48.5]], [[75.0]], [[33.0]], [[56.0]], [[0.0]], [[0.0]], [[38.5]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[23.375]], [[6.993104012531504e-18]], [[54.25]], [[0.0]], [[49.75]], [[0.0]], [[0.0]]]}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.32421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.455078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[51.75]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.45.feed_forward.shared_expert.down_proj": {"inputs": [[[222.0]]], "outputs": [[[28.0]], [[1.42227395548986e+38]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.46.self_attn.qkv_proj": {"inputs": [[[26.75]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.46.self_attn.o_proj": {"inputs": [[[20.125]]], "outputs": [[[101.0]], [[188.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.46.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.671875]], [[16.5]]]}, "language_model.model.layers.46.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[21.75]]]}, "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.671875]]]}, "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[19.875]]]}, "language_model.model.layers.46.self_attn.attn.impl.k_cache": {"inputs": [[[16.5]]]}, "language_model.model.layers.46.self_attn.attn.impl.v_cache": {"inputs": [[[21.75]]]}, "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[20.0]], [[16.5]], [[21.75]]], "outputs": [[[19.5]], [[1.0]]]}, "language_model.model.layers.46.feed_forward.gate_up_proj": {"inputs": [[[40.5]]], "params": {"weight": [[0.55859375]]}}, "language_model.model.layers.46.feed_forward.down_proj": {"inputs": [[[318.0]]], "outputs": [[[1688.0]], [[2400.0]]], "params": {"weight": [[0.5859375]]}}, "language_model.model.layers.47.self_attn.qkv_proj": {"inputs": [[[19.875]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.47.self_attn.o_proj": {"inputs": [[[5.6875]]], "outputs": [[[98.0]], [[126.5]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.47.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.427734375]], [[11.9375]]]}, "language_model.model.layers.47.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.75]]]}, "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.427734375]]]}, "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[8.1875]]]}, "language_model.model.layers.47.self_attn.attn.impl.k_cache": {"inputs": [[[11.9375]]]}, "language_model.model.layers.47.self_attn.attn.impl.v_cache": {"inputs": [[[9.75]]]}, "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[5.375]], [[11.9375]], [[9.75]]], "outputs": [[[5.6875]], [[1.0]]]}, "language_model.model.layers.47.feed_forward.router": {"inputs": [[[33.5]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op": {"inputs": [[[33.5]]], "outputs": [[[187.0]], [[94.5]], [[61.0]], [[55.5]], [[226.0]], [[66.5]], [[29.75]], [[33.25]], [[60.5]], [[89.0]], [[244.0]], [[54.0]], [[176.0]], [[96.5]], [[40.5]], [[75.0]], [[37.0]], [[45.5]], [[50.5]], [[50.0]], [[67.0]], [[83.5]], [[152.0]], [[44.0]], [[50.0]], [[179.0]], [[282.0]], [[30.875]], [[31.5]], [[29.875]], [[72.5]], [[51.75]], [[54.0]], [[23.625]], [[50.25]], [[195.0]], [[31.25]], [[31.875]], [[21.75]], [[294.0]], [[155.0]], [[27.5]], [[116.0]], [[127.0]], [[27.0]], [[59.5]], [[69.5]], [[230.0]], [[50.5]], [[24.125]], [[114.5]], [[101.5]], [[306.0]], [[95.0]], [[39.0]], [[32.0]], [[37.25]], [[74.5]], [[61.5]], [[32.75]], [[37.0]], [[50.5]], [[96.5]], [[127.0]], [[157.0]], [[38.75]], [[30.625]], [[80.0]], [[207.0]], [[28.0]], [[198.0]], [[163.0]], [[27.625]], [[31.125]], [[33.75]], [[82.5]], [[26.5]], [[44.0]], [[57.25]], [[190.0]], [[25.0]], [[109.5]], [[246.0]], [[306.0]], [[38.75]], [[37.25]], [[183.0]], [[112.0]], [[39.75]], [[32.75]], [[48.0]], [[131.0]], [[218.0]], [[220.0]], [[45.0]], [[88.5]], [[95.0]], [[148.0]], [[37.75]], [[29.5]], [[57.25]], [[66.5]], [[43.5]], [[90.0]], [[40.25]], [[103.5]], [[34.75]], [[63.0]], [[324.0]], [[30.875]], [[112.0]], [[193.0]], [[151.0]], [[72.0]], [[59.25]], [[83.5]], [[70.5]], [[164.0]], [[159.0]], [[64.5]], [[304.0]], [[112.5]], [[63.0]], [[90.0]], [[30.25]], [[50.25]], [[138.0]], [[106.5]], [[179.0]]]}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.361328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.447265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.4140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.365234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.51171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.66015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.54296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.5234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.4609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.68359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.58203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.51953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.431640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.330078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.458984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.81640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.51171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.490234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.388671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.482421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.85546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.61328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.75]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.388671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.53125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.3046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.9140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.3984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.423828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.68359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.4921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.6328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.77734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.435546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.8203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.83984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.6484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.3984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.38671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.478515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.5234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.392578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.36328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.357421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.78515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.5859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.458984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.427734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.76171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.3984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.55859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.78515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.76953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.5859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.35546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.8203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.60546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.65234375]]}}, "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[33.5]]], "params": {"weight": [[0.390625]]}}, "language_model.model.layers.47.feed_forward.shared_expert.down_proj": {"inputs": [[[1152.0]]], "outputs": [[[632.0]], [[2.9907629905160607e+38]]], "params": {"weight": [[0.40625]]}}, "language_model.lm_head": {"inputs": [[[52.0]]], "params": {"weight": [[1.28125]]}}}} \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_6_8.npz b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_6_8.npz new file mode 100644 index 000000000000..a5274af88bac Binary files /dev/null and b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_6_8.npz differ diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_6_8_mod_list.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_6_8_mod_list.json new file mode 100644 index 000000000000..928cd9e2a2c5 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_6_8_mod_list.json @@ -0,0 +1,6935 @@ +[ + "vision_model.patch_embedding.linear", + "vision_model.model.layers.0.self_attn.qkv_proj", + "vision_model.model.layers.0.self_attn.o_proj", + "vision_model.model.layers.0.mlp.fc1", + "vision_model.model.layers.0.mlp.fc2", + "vision_model.model.layers.1.self_attn.qkv_proj", + "vision_model.model.layers.1.self_attn.o_proj", + "vision_model.model.layers.1.mlp.fc1", + "vision_model.model.layers.1.mlp.fc2", + "vision_model.model.layers.2.self_attn.qkv_proj", + "vision_model.model.layers.2.self_attn.o_proj", + "vision_model.model.layers.2.mlp.fc1", + "vision_model.model.layers.2.mlp.fc2", + "vision_model.model.layers.3.self_attn.qkv_proj", + "vision_model.model.layers.3.self_attn.o_proj", + "vision_model.model.layers.3.mlp.fc1", + "vision_model.model.layers.3.mlp.fc2", + "vision_model.model.layers.4.self_attn.qkv_proj", + "vision_model.model.layers.4.self_attn.o_proj", + "vision_model.model.layers.4.mlp.fc1", + "vision_model.model.layers.4.mlp.fc2", + "vision_model.model.layers.5.self_attn.qkv_proj", + "vision_model.model.layers.5.self_attn.o_proj", + "vision_model.model.layers.5.mlp.fc1", + "vision_model.model.layers.5.mlp.fc2", + "vision_model.model.layers.6.self_attn.qkv_proj", + "vision_model.model.layers.6.self_attn.o_proj", + "vision_model.model.layers.6.mlp.fc1", + "vision_model.model.layers.6.mlp.fc2", + "vision_model.model.layers.7.self_attn.qkv_proj", + "vision_model.model.layers.7.self_attn.o_proj", + "vision_model.model.layers.7.mlp.fc1", + "vision_model.model.layers.7.mlp.fc2", + "vision_model.model.layers.8.self_attn.qkv_proj", + "vision_model.model.layers.8.self_attn.o_proj", + "vision_model.model.layers.8.mlp.fc1", + "vision_model.model.layers.8.mlp.fc2", + "vision_model.model.layers.9.self_attn.qkv_proj", + "vision_model.model.layers.9.self_attn.o_proj", + "vision_model.model.layers.9.mlp.fc1", + "vision_model.model.layers.9.mlp.fc2", + "vision_model.model.layers.10.self_attn.qkv_proj", + "vision_model.model.layers.10.self_attn.o_proj", + "vision_model.model.layers.10.mlp.fc1", + "vision_model.model.layers.10.mlp.fc2", + "vision_model.model.layers.11.self_attn.qkv_proj", + "vision_model.model.layers.11.self_attn.o_proj", + "vision_model.model.layers.11.mlp.fc1", + "vision_model.model.layers.11.mlp.fc2", + "vision_model.model.layers.12.self_attn.qkv_proj", + "vision_model.model.layers.12.self_attn.o_proj", + "vision_model.model.layers.12.mlp.fc1", + "vision_model.model.layers.12.mlp.fc2", + "vision_model.model.layers.13.self_attn.qkv_proj", + "vision_model.model.layers.13.self_attn.o_proj", + "vision_model.model.layers.13.mlp.fc1", + "vision_model.model.layers.13.mlp.fc2", + "vision_model.model.layers.14.self_attn.qkv_proj", + "vision_model.model.layers.14.self_attn.o_proj", + "vision_model.model.layers.14.mlp.fc1", + "vision_model.model.layers.14.mlp.fc2", + "vision_model.model.layers.15.self_attn.qkv_proj", + "vision_model.model.layers.15.self_attn.o_proj", + "vision_model.model.layers.15.mlp.fc1", + "vision_model.model.layers.15.mlp.fc2", + "vision_model.model.layers.16.self_attn.qkv_proj", + "vision_model.model.layers.16.self_attn.o_proj", + "vision_model.model.layers.16.mlp.fc1", + "vision_model.model.layers.16.mlp.fc2", + "vision_model.model.layers.17.self_attn.qkv_proj", + "vision_model.model.layers.17.self_attn.o_proj", + "vision_model.model.layers.17.mlp.fc1", + "vision_model.model.layers.17.mlp.fc2", + "vision_model.model.layers.18.self_attn.qkv_proj", + "vision_model.model.layers.18.self_attn.o_proj", + "vision_model.model.layers.18.mlp.fc1", + "vision_model.model.layers.18.mlp.fc2", + "vision_model.model.layers.19.self_attn.qkv_proj", + "vision_model.model.layers.19.self_attn.o_proj", + "vision_model.model.layers.19.mlp.fc1", + "vision_model.model.layers.19.mlp.fc2", + "vision_model.model.layers.20.self_attn.qkv_proj", + "vision_model.model.layers.20.self_attn.o_proj", + "vision_model.model.layers.20.mlp.fc1", + "vision_model.model.layers.20.mlp.fc2", + "vision_model.model.layers.21.self_attn.qkv_proj", + "vision_model.model.layers.21.self_attn.o_proj", + "vision_model.model.layers.21.mlp.fc1", + "vision_model.model.layers.21.mlp.fc2", + "vision_model.model.layers.22.self_attn.qkv_proj", + "vision_model.model.layers.22.self_attn.o_proj", + "vision_model.model.layers.22.mlp.fc1", + "vision_model.model.layers.22.mlp.fc2", + "vision_model.model.layers.23.self_attn.qkv_proj", + "vision_model.model.layers.23.self_attn.o_proj", + "vision_model.model.layers.23.mlp.fc1", + "vision_model.model.layers.23.mlp.fc2", + "vision_model.model.layers.24.self_attn.qkv_proj", + "vision_model.model.layers.24.self_attn.o_proj", + "vision_model.model.layers.24.mlp.fc1", + "vision_model.model.layers.24.mlp.fc2", + "vision_model.model.layers.25.self_attn.qkv_proj", + "vision_model.model.layers.25.self_attn.o_proj", + "vision_model.model.layers.25.mlp.fc1", + "vision_model.model.layers.25.mlp.fc2", + "vision_model.model.layers.26.self_attn.qkv_proj", + "vision_model.model.layers.26.self_attn.o_proj", + "vision_model.model.layers.26.mlp.fc1", + "vision_model.model.layers.26.mlp.fc2", + "vision_model.model.layers.27.self_attn.qkv_proj", + "vision_model.model.layers.27.self_attn.o_proj", + "vision_model.model.layers.27.mlp.fc1", + "vision_model.model.layers.27.mlp.fc2", + "vision_model.model.layers.28.self_attn.qkv_proj", + "vision_model.model.layers.28.self_attn.o_proj", + "vision_model.model.layers.28.mlp.fc1", + "vision_model.model.layers.28.mlp.fc2", + "vision_model.model.layers.29.self_attn.qkv_proj", + "vision_model.model.layers.29.self_attn.o_proj", + "vision_model.model.layers.29.mlp.fc1", + "vision_model.model.layers.29.mlp.fc2", + "vision_model.model.layers.30.self_attn.qkv_proj", + "vision_model.model.layers.30.self_attn.o_proj", + "vision_model.model.layers.30.mlp.fc1", + "vision_model.model.layers.30.mlp.fc2", + "vision_model.model.layers.31.self_attn.qkv_proj", + "vision_model.model.layers.31.self_attn.o_proj", + "vision_model.model.layers.31.mlp.fc1", + "vision_model.model.layers.31.mlp.fc2", + "vision_model.model.layers.32.self_attn.qkv_proj", + "vision_model.model.layers.32.self_attn.o_proj", + "vision_model.model.layers.32.mlp.fc1", + "vision_model.model.layers.32.mlp.fc2", + "vision_model.model.layers.33.self_attn.qkv_proj", + "vision_model.model.layers.33.self_attn.o_proj", + "vision_model.model.layers.33.mlp.fc1", + "vision_model.model.layers.33.mlp.fc2", + "vision_model.vision_adapter.mlp.fc1", + "vision_model.vision_adapter.mlp.fc2", + "multi_modal_projector.linear_1", + "language_model.model.layers.0.self_attn.qkv_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.0.self_attn.attn.impl.softmax", + "language_model.model.layers.0.self_attn.attn.impl.matmul_av", + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.0.self_attn.attn.impl.k_cache", + "language_model.model.layers.0.self_attn.attn.impl.v_cache", + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.0.feed_forward.gate_up_proj", + "language_model.model.layers.0.feed_forward.down_proj", + "language_model.model.layers.1.self_attn.qkv_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.1.self_attn.attn.impl.softmax", + "language_model.model.layers.1.self_attn.attn.impl.matmul_av", + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.1.self_attn.attn.impl.k_cache", + "language_model.model.layers.1.self_attn.attn.impl.v_cache", + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.1.feed_forward.router", + "language_model.model.layers.1.feed_forward.experts", + "language_model.model.layers.1.feed_forward.experts.moe_op", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.1.feed_forward.shared_expert.down_proj", + "language_model.model.layers.2.self_attn.qkv_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.2.self_attn.attn.impl.softmax", + "language_model.model.layers.2.self_attn.attn.impl.matmul_av", + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.2.self_attn.attn.impl.k_cache", + "language_model.model.layers.2.self_attn.attn.impl.v_cache", + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.2.feed_forward.gate_up_proj", + "language_model.model.layers.2.feed_forward.down_proj", + "language_model.model.layers.3.self_attn.qkv_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.3.self_attn.attn.impl.softmax", + "language_model.model.layers.3.self_attn.attn.impl.matmul_av", + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.3.self_attn.attn.impl.k_cache", + "language_model.model.layers.3.self_attn.attn.impl.v_cache", + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.3.feed_forward.router", + "language_model.model.layers.3.feed_forward.experts", + "language_model.model.layers.3.feed_forward.experts.moe_op", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.3.feed_forward.shared_expert.down_proj", + "language_model.model.layers.4.self_attn.qkv_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.4.self_attn.attn.impl.softmax", + "language_model.model.layers.4.self_attn.attn.impl.matmul_av", + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.4.self_attn.attn.impl.k_cache", + "language_model.model.layers.4.self_attn.attn.impl.v_cache", + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.4.feed_forward.gate_up_proj", + "language_model.model.layers.4.feed_forward.down_proj", + "language_model.model.layers.5.self_attn.qkv_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.5.self_attn.attn.impl.softmax", + "language_model.model.layers.5.self_attn.attn.impl.matmul_av", + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.5.self_attn.attn.impl.k_cache", + "language_model.model.layers.5.self_attn.attn.impl.v_cache", + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.5.feed_forward.router", + "language_model.model.layers.5.feed_forward.experts", + "language_model.model.layers.5.feed_forward.experts.moe_op", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.5.feed_forward.shared_expert.down_proj", + "language_model.model.layers.6.self_attn.qkv_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.6.self_attn.attn.impl.softmax", + "language_model.model.layers.6.self_attn.attn.impl.matmul_av", + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.6.self_attn.attn.impl.k_cache", + "language_model.model.layers.6.self_attn.attn.impl.v_cache", + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.6.feed_forward.gate_up_proj", + "language_model.model.layers.6.feed_forward.down_proj", + "language_model.model.layers.7.self_attn.qkv_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.7.self_attn.attn.impl.softmax", + "language_model.model.layers.7.self_attn.attn.impl.matmul_av", + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.7.self_attn.attn.impl.k_cache", + "language_model.model.layers.7.self_attn.attn.impl.v_cache", + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.7.feed_forward.router", + "language_model.model.layers.7.feed_forward.experts", + "language_model.model.layers.7.feed_forward.experts.moe_op", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.7.feed_forward.shared_expert.down_proj", + "language_model.model.layers.8.self_attn.qkv_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.8.self_attn.attn.impl.softmax", + "language_model.model.layers.8.self_attn.attn.impl.matmul_av", + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.8.self_attn.attn.impl.k_cache", + "language_model.model.layers.8.self_attn.attn.impl.v_cache", + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.8.feed_forward.gate_up_proj", + "language_model.model.layers.8.feed_forward.down_proj", + "language_model.model.layers.9.self_attn.qkv_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.9.self_attn.attn.impl.softmax", + "language_model.model.layers.9.self_attn.attn.impl.matmul_av", + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.9.self_attn.attn.impl.k_cache", + "language_model.model.layers.9.self_attn.attn.impl.v_cache", + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.9.feed_forward.router", + "language_model.model.layers.9.feed_forward.experts", + "language_model.model.layers.9.feed_forward.experts.moe_op", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.9.feed_forward.shared_expert.down_proj", + "language_model.model.layers.10.self_attn.qkv_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.10.self_attn.attn.impl.softmax", + "language_model.model.layers.10.self_attn.attn.impl.matmul_av", + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.10.self_attn.attn.impl.k_cache", + "language_model.model.layers.10.self_attn.attn.impl.v_cache", + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.10.feed_forward.gate_up_proj", + "language_model.model.layers.10.feed_forward.down_proj", + "language_model.model.layers.11.self_attn.qkv_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.11.self_attn.attn.impl.softmax", + "language_model.model.layers.11.self_attn.attn.impl.matmul_av", + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.11.self_attn.attn.impl.k_cache", + "language_model.model.layers.11.self_attn.attn.impl.v_cache", + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.11.feed_forward.router", + "language_model.model.layers.11.feed_forward.experts", + "language_model.model.layers.11.feed_forward.experts.moe_op", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.11.feed_forward.shared_expert.down_proj", + "language_model.model.layers.12.self_attn.qkv_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.12.self_attn.attn.impl.softmax", + "language_model.model.layers.12.self_attn.attn.impl.matmul_av", + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.12.self_attn.attn.impl.k_cache", + "language_model.model.layers.12.self_attn.attn.impl.v_cache", + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.12.feed_forward.gate_up_proj", + "language_model.model.layers.12.feed_forward.down_proj", + "language_model.model.layers.13.self_attn.qkv_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.13.self_attn.attn.impl.softmax", + "language_model.model.layers.13.self_attn.attn.impl.matmul_av", + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.13.self_attn.attn.impl.k_cache", + "language_model.model.layers.13.self_attn.attn.impl.v_cache", + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.13.feed_forward.router", + "language_model.model.layers.13.feed_forward.experts", + "language_model.model.layers.13.feed_forward.experts.moe_op", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.13.feed_forward.shared_expert.down_proj", + "language_model.model.layers.14.self_attn.qkv_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.14.self_attn.attn.impl.softmax", + "language_model.model.layers.14.self_attn.attn.impl.matmul_av", + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.14.self_attn.attn.impl.k_cache", + "language_model.model.layers.14.self_attn.attn.impl.v_cache", + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.14.feed_forward.gate_up_proj", + "language_model.model.layers.14.feed_forward.down_proj", + "language_model.model.layers.15.self_attn.qkv_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.15.self_attn.attn.impl.softmax", + "language_model.model.layers.15.self_attn.attn.impl.matmul_av", + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.15.self_attn.attn.impl.k_cache", + "language_model.model.layers.15.self_attn.attn.impl.v_cache", + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.15.feed_forward.router", + "language_model.model.layers.15.feed_forward.experts", + "language_model.model.layers.15.feed_forward.experts.moe_op", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.15.feed_forward.shared_expert.down_proj", + "language_model.model.layers.16.self_attn.qkv_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.16.self_attn.attn.impl.softmax", + "language_model.model.layers.16.self_attn.attn.impl.matmul_av", + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.16.self_attn.attn.impl.k_cache", + "language_model.model.layers.16.self_attn.attn.impl.v_cache", + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.16.feed_forward.gate_up_proj", + "language_model.model.layers.16.feed_forward.down_proj", + "language_model.model.layers.17.self_attn.qkv_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.17.self_attn.attn.impl.softmax", + "language_model.model.layers.17.self_attn.attn.impl.matmul_av", + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.17.self_attn.attn.impl.k_cache", + "language_model.model.layers.17.self_attn.attn.impl.v_cache", + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.17.feed_forward.router", + "language_model.model.layers.17.feed_forward.experts", + "language_model.model.layers.17.feed_forward.experts.moe_op", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.17.feed_forward.shared_expert.down_proj", + "language_model.model.layers.18.self_attn.qkv_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.18.self_attn.attn.impl.softmax", + "language_model.model.layers.18.self_attn.attn.impl.matmul_av", + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.18.self_attn.attn.impl.k_cache", + "language_model.model.layers.18.self_attn.attn.impl.v_cache", + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.18.feed_forward.gate_up_proj", + "language_model.model.layers.18.feed_forward.down_proj", + "language_model.model.layers.19.self_attn.qkv_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.19.self_attn.attn.impl.softmax", + "language_model.model.layers.19.self_attn.attn.impl.matmul_av", + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.19.self_attn.attn.impl.k_cache", + "language_model.model.layers.19.self_attn.attn.impl.v_cache", + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.19.feed_forward.router", + "language_model.model.layers.19.feed_forward.experts", + "language_model.model.layers.19.feed_forward.experts.moe_op", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.19.feed_forward.shared_expert.down_proj", + "language_model.model.layers.20.self_attn.qkv_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.20.self_attn.attn.impl.softmax", + "language_model.model.layers.20.self_attn.attn.impl.matmul_av", + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.20.self_attn.attn.impl.k_cache", + "language_model.model.layers.20.self_attn.attn.impl.v_cache", + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.20.feed_forward.gate_up_proj", + "language_model.model.layers.20.feed_forward.down_proj", + "language_model.model.layers.21.self_attn.qkv_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.21.self_attn.attn.impl.softmax", + "language_model.model.layers.21.self_attn.attn.impl.matmul_av", + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.21.self_attn.attn.impl.k_cache", + "language_model.model.layers.21.self_attn.attn.impl.v_cache", + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.21.feed_forward.router", + "language_model.model.layers.21.feed_forward.experts", + "language_model.model.layers.21.feed_forward.experts.moe_op", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.21.feed_forward.shared_expert.down_proj", + "language_model.model.layers.22.self_attn.qkv_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.22.self_attn.attn.impl.softmax", + "language_model.model.layers.22.self_attn.attn.impl.matmul_av", + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.22.self_attn.attn.impl.k_cache", + "language_model.model.layers.22.self_attn.attn.impl.v_cache", + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.22.feed_forward.gate_up_proj", + "language_model.model.layers.22.feed_forward.down_proj", + "language_model.model.layers.23.self_attn.qkv_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.23.self_attn.attn.impl.softmax", + "language_model.model.layers.23.self_attn.attn.impl.matmul_av", + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.23.self_attn.attn.impl.k_cache", + "language_model.model.layers.23.self_attn.attn.impl.v_cache", + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.23.feed_forward.router", + "language_model.model.layers.23.feed_forward.experts", + "language_model.model.layers.23.feed_forward.experts.moe_op", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.23.feed_forward.shared_expert.down_proj", + "language_model.model.layers.24.self_attn.qkv_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.24.self_attn.attn.impl.softmax", + "language_model.model.layers.24.self_attn.attn.impl.matmul_av", + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.24.self_attn.attn.impl.k_cache", + "language_model.model.layers.24.self_attn.attn.impl.v_cache", + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.24.feed_forward.gate_up_proj", + "language_model.model.layers.24.feed_forward.down_proj", + "language_model.model.layers.25.self_attn.qkv_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.25.self_attn.attn.impl.softmax", + "language_model.model.layers.25.self_attn.attn.impl.matmul_av", + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.25.self_attn.attn.impl.k_cache", + "language_model.model.layers.25.self_attn.attn.impl.v_cache", + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.25.feed_forward.router", + "language_model.model.layers.25.feed_forward.experts", + "language_model.model.layers.25.feed_forward.experts.moe_op", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.25.feed_forward.shared_expert.down_proj", + "language_model.model.layers.26.self_attn.qkv_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.26.self_attn.attn.impl.softmax", + "language_model.model.layers.26.self_attn.attn.impl.matmul_av", + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.26.self_attn.attn.impl.k_cache", + "language_model.model.layers.26.self_attn.attn.impl.v_cache", + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.26.feed_forward.gate_up_proj", + "language_model.model.layers.26.feed_forward.down_proj", + "language_model.model.layers.27.self_attn.qkv_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.27.self_attn.attn.impl.softmax", + "language_model.model.layers.27.self_attn.attn.impl.matmul_av", + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.27.self_attn.attn.impl.k_cache", + "language_model.model.layers.27.self_attn.attn.impl.v_cache", + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.27.feed_forward.router", + "language_model.model.layers.27.feed_forward.experts", + "language_model.model.layers.27.feed_forward.experts.moe_op", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.27.feed_forward.shared_expert.down_proj", + "language_model.model.layers.28.self_attn.qkv_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.28.self_attn.attn.impl.softmax", + "language_model.model.layers.28.self_attn.attn.impl.matmul_av", + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.28.self_attn.attn.impl.k_cache", + "language_model.model.layers.28.self_attn.attn.impl.v_cache", + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.28.feed_forward.gate_up_proj", + "language_model.model.layers.28.feed_forward.down_proj", + "language_model.model.layers.29.self_attn.qkv_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.29.self_attn.attn.impl.softmax", + "language_model.model.layers.29.self_attn.attn.impl.matmul_av", + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.29.self_attn.attn.impl.k_cache", + "language_model.model.layers.29.self_attn.attn.impl.v_cache", + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.29.feed_forward.router", + "language_model.model.layers.29.feed_forward.experts", + "language_model.model.layers.29.feed_forward.experts.moe_op", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.29.feed_forward.shared_expert.down_proj", + "language_model.model.layers.30.self_attn.qkv_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.30.self_attn.attn.impl.softmax", + "language_model.model.layers.30.self_attn.attn.impl.matmul_av", + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.30.self_attn.attn.impl.k_cache", + "language_model.model.layers.30.self_attn.attn.impl.v_cache", + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.30.feed_forward.gate_up_proj", + "language_model.model.layers.30.feed_forward.down_proj", + "language_model.model.layers.31.self_attn.qkv_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.31.self_attn.attn.impl.softmax", + "language_model.model.layers.31.self_attn.attn.impl.matmul_av", + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.31.self_attn.attn.impl.k_cache", + "language_model.model.layers.31.self_attn.attn.impl.v_cache", + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.31.feed_forward.router", + "language_model.model.layers.31.feed_forward.experts", + "language_model.model.layers.31.feed_forward.experts.moe_op", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.31.feed_forward.shared_expert.down_proj", + "language_model.model.layers.32.self_attn.qkv_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.32.self_attn.attn.impl.softmax", + "language_model.model.layers.32.self_attn.attn.impl.matmul_av", + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.32.self_attn.attn.impl.k_cache", + "language_model.model.layers.32.self_attn.attn.impl.v_cache", + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.32.feed_forward.gate_up_proj", + "language_model.model.layers.32.feed_forward.down_proj", + "language_model.model.layers.33.self_attn.qkv_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.33.self_attn.attn.impl.softmax", + "language_model.model.layers.33.self_attn.attn.impl.matmul_av", + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.33.self_attn.attn.impl.k_cache", + "language_model.model.layers.33.self_attn.attn.impl.v_cache", + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.33.feed_forward.router", + "language_model.model.layers.33.feed_forward.experts", + "language_model.model.layers.33.feed_forward.experts.moe_op", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.33.feed_forward.shared_expert.down_proj", + "language_model.model.layers.34.self_attn.qkv_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.34.self_attn.attn.impl.softmax", + "language_model.model.layers.34.self_attn.attn.impl.matmul_av", + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.34.self_attn.attn.impl.k_cache", + "language_model.model.layers.34.self_attn.attn.impl.v_cache", + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.34.feed_forward.gate_up_proj", + "language_model.model.layers.34.feed_forward.down_proj", + "language_model.model.layers.35.self_attn.qkv_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.35.self_attn.attn.impl.softmax", + "language_model.model.layers.35.self_attn.attn.impl.matmul_av", + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.35.self_attn.attn.impl.k_cache", + "language_model.model.layers.35.self_attn.attn.impl.v_cache", + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.35.feed_forward.router", + "language_model.model.layers.35.feed_forward.experts", + "language_model.model.layers.35.feed_forward.experts.moe_op", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.35.feed_forward.shared_expert.down_proj", + "language_model.model.layers.36.self_attn.qkv_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.36.self_attn.attn.impl.softmax", + "language_model.model.layers.36.self_attn.attn.impl.matmul_av", + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.36.self_attn.attn.impl.k_cache", + "language_model.model.layers.36.self_attn.attn.impl.v_cache", + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.36.feed_forward.gate_up_proj", + "language_model.model.layers.36.feed_forward.down_proj", + "language_model.model.layers.37.self_attn.qkv_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.37.self_attn.attn.impl.softmax", + "language_model.model.layers.37.self_attn.attn.impl.matmul_av", + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.37.self_attn.attn.impl.k_cache", + "language_model.model.layers.37.self_attn.attn.impl.v_cache", + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.37.feed_forward.router", + "language_model.model.layers.37.feed_forward.experts", + "language_model.model.layers.37.feed_forward.experts.moe_op", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.37.feed_forward.shared_expert.down_proj", + "language_model.model.layers.38.self_attn.qkv_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.38.self_attn.attn.impl.softmax", + "language_model.model.layers.38.self_attn.attn.impl.matmul_av", + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.38.self_attn.attn.impl.k_cache", + "language_model.model.layers.38.self_attn.attn.impl.v_cache", + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.38.feed_forward.gate_up_proj", + "language_model.model.layers.38.feed_forward.down_proj", + "language_model.model.layers.39.self_attn.qkv_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.39.self_attn.attn.impl.softmax", + "language_model.model.layers.39.self_attn.attn.impl.matmul_av", + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.39.self_attn.attn.impl.k_cache", + "language_model.model.layers.39.self_attn.attn.impl.v_cache", + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.39.feed_forward.router", + "language_model.model.layers.39.feed_forward.experts", + "language_model.model.layers.39.feed_forward.experts.moe_op", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.39.feed_forward.shared_expert.down_proj", + "language_model.model.layers.40.self_attn.qkv_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.40.self_attn.attn.impl.softmax", + "language_model.model.layers.40.self_attn.attn.impl.matmul_av", + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.40.self_attn.attn.impl.k_cache", + "language_model.model.layers.40.self_attn.attn.impl.v_cache", + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.40.feed_forward.gate_up_proj", + "language_model.model.layers.40.feed_forward.down_proj", + "language_model.model.layers.41.self_attn.qkv_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.41.self_attn.attn.impl.softmax", + "language_model.model.layers.41.self_attn.attn.impl.matmul_av", + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.41.self_attn.attn.impl.k_cache", + "language_model.model.layers.41.self_attn.attn.impl.v_cache", + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.41.feed_forward.router", + "language_model.model.layers.41.feed_forward.experts", + "language_model.model.layers.41.feed_forward.experts.moe_op", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.41.feed_forward.shared_expert.down_proj", + "language_model.model.layers.42.self_attn.qkv_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.42.self_attn.attn.impl.softmax", + "language_model.model.layers.42.self_attn.attn.impl.matmul_av", + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.42.self_attn.attn.impl.k_cache", + "language_model.model.layers.42.self_attn.attn.impl.v_cache", + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.42.feed_forward.gate_up_proj", + "language_model.model.layers.42.feed_forward.down_proj", + "language_model.model.layers.43.self_attn.qkv_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.43.self_attn.attn.impl.softmax", + "language_model.model.layers.43.self_attn.attn.impl.matmul_av", + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.43.self_attn.attn.impl.k_cache", + "language_model.model.layers.43.self_attn.attn.impl.v_cache", + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.43.feed_forward.router", + "language_model.model.layers.43.feed_forward.experts", + "language_model.model.layers.43.feed_forward.experts.moe_op", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.43.feed_forward.shared_expert.down_proj", + "language_model.model.layers.44.self_attn.qkv_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.44.self_attn.attn.impl.softmax", + "language_model.model.layers.44.self_attn.attn.impl.matmul_av", + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.44.self_attn.attn.impl.k_cache", + "language_model.model.layers.44.self_attn.attn.impl.v_cache", + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.44.feed_forward.gate_up_proj", + "language_model.model.layers.44.feed_forward.down_proj", + "language_model.model.layers.45.self_attn.qkv_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.45.self_attn.attn.impl.softmax", + "language_model.model.layers.45.self_attn.attn.impl.matmul_av", + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.45.self_attn.attn.impl.k_cache", + "language_model.model.layers.45.self_attn.attn.impl.v_cache", + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.45.feed_forward.router", + "language_model.model.layers.45.feed_forward.experts", + "language_model.model.layers.45.feed_forward.experts.moe_op", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.45.feed_forward.shared_expert.down_proj", + "language_model.model.layers.46.self_attn.qkv_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.46.self_attn.attn.impl.softmax", + "language_model.model.layers.46.self_attn.attn.impl.matmul_av", + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.46.self_attn.attn.impl.k_cache", + "language_model.model.layers.46.self_attn.attn.impl.v_cache", + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.46.feed_forward.gate_up_proj", + "language_model.model.layers.46.feed_forward.down_proj", + "language_model.model.layers.47.self_attn.qkv_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.47.self_attn.attn.impl.softmax", + "language_model.model.layers.47.self_attn.attn.impl.matmul_av", + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.47.self_attn.attn.impl.k_cache", + "language_model.model.layers.47.self_attn.attn.impl.v_cache", + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.47.feed_forward.router", + "language_model.model.layers.47.feed_forward.experts", + "language_model.model.layers.47.feed_forward.experts.moe_op", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.47.feed_forward.shared_expert.down_proj", + "language_model.lm_head" +] \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_7_8.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_7_8.json new file mode 100644 index 000000000000..3cc1d016bd8e --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_7_8.json @@ -0,0 +1 @@ +{"GlobalRank": null, "LocalRank": 7, "Mode": "DynamicRange", "Nodes": {"language_model.model.layers.0.self_attn.qkv_proj": {"inputs": [[[11.1875]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.0.self_attn.o_proj": {"inputs": [[[0.9375]]], "outputs": [[[0.80078125]], [[3.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.0.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.68359375]], [[6.75]]]}, "language_model.model.layers.0.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.9609375]]]}, "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.68359375]]]}, "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.9609375]]]}, "language_model.model.layers.0.self_attn.attn.impl.k_cache": {"inputs": [[[6.75]]]}, "language_model.model.layers.0.self_attn.attn.impl.v_cache": {"inputs": [[[0.9609375]]]}, "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[7.9375]], [[6.75]], [[0.9609375]]], "outputs": [[[0.9375]], [[1.0]]]}, "language_model.model.layers.0.feed_forward.gate_up_proj": {"inputs": [[[2.546875]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.0.feed_forward.down_proj": {"inputs": [[[3.65625]]], "outputs": [[[26.75]], [[65.0]]], "params": {"weight": [[0.66015625]]}}, "language_model.model.layers.1.self_attn.qkv_proj": {"inputs": [[[12.3125]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.1.self_attn.o_proj": {"inputs": [[[1.15625]]], "outputs": [[[0.52734375]], [[1.0078125]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.1.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.703125]], [[10.125]]]}, "language_model.model.layers.1.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.4609375]]]}, "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.703125]]]}, "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.390625]]]}, "language_model.model.layers.1.self_attn.attn.impl.k_cache": {"inputs": [[[10.125]]]}, "language_model.model.layers.1.self_attn.attn.impl.v_cache": {"inputs": [[[1.4609375]]]}, "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[7.125]], [[9.75]], [[1.4609375]]], "outputs": [[[1.15625]], [[1.0]]]}, "language_model.model.layers.1.feed_forward.router": {"inputs": [[[1.8828125]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op": {"inputs": [[[1.8828125]]], "outputs": [[[0.9921875]], [[8.875]], [[0.0]], [[0.765625]], [[14.75]], [[14.5625]], [[18.75]], [[11.1875]], [[11.375]], [[4.8125]], [[13.625]], [[12.0]], [[12.5]], [[4.9375]], [[9.6875]], [[10.8125]], [[19.75]], [[2.03125]], [[5.3125]], [[10.4375]], [[13.0]], [[14.3125]], [[9.0625]], [[0.0]], [[10.9375]], [[5.59375]], [[1.3359375]], [[13.1875]], [[10.8125]], [[7.84375]], [[10.0]], [[14.375]], [[18.0]], [[0.1845703125]], [[4.75]], [[0.0]], [[10.3125]], [[1.75]], [[12.625]], [[5.03125]], [[12.9375]], [[14.625]], [[12.3125]], [[8.8125]], [[12.0625]], [[1.65625]], [[16.5]], [[9.6875]], [[6.28125]], [[3.46875]], [[0.0]], [[8.625]], [[11.1875]], [[7.5625]], [[7.375]], [[13.625]], [[14.0]], [[14.0]], [[0.5]], [[9.75]], [[9.25]], [[11.6875]], [[15.8125]], [[12.5625]], [[0.0]], [[14.125]], [[6.03125]], [[2.09375]], [[9.8125]], [[6.71875]], [[4.625]], [[10.75]], [[11.5625]], [[13.0625]], [[14.25]], [[10.8125]], [[8.0625]], [[13.0]], [[11.5]], [[11.8125]], [[14.875]], [[0.0]], [[0.875]], [[15.9375]], [[2.109375]], [[15.4375]], [[11.9375]], [[11.4375]], [[11.8125]], [[5.5625]], [[11.0625]], [[13.9375]], [[9.125]], [[13.0625]], [[0.0]], [[14.4375]], [[7.21875]], [[0.08984375]], [[0.0]], [[11.0625]], [[10.25]], [[8.75]], [[2.5625]], [[11.3125]], [[5.625]], [[2.171875]], [[11.625]], [[0.150390625]], [[2.09375]], [[2.828125]], [[0.322265625]], [[10.5625]], [[0.64453125]], [[11.8125]], [[0.0]], [[13.5625]], [[15.875]], [[12.5625]], [[2.546875]], [[14.125]], [[5.75]], [[10.3125]], [[12.4375]], [[2.34375]], [[10.5625]], [[8.5]], [[3.484375]], [[12.5625]], [[13.375]]]}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.431640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.361328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.32421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.5703125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[1.8828125]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.1.feed_forward.shared_expert.down_proj": {"inputs": [[[4.3125]]], "outputs": [[[0.62890625]], [[1.6096120261457966e+35]]], "params": {"weight": [[0.43359375]]}}, "language_model.model.layers.2.self_attn.qkv_proj": {"inputs": [[[9.875]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.2.self_attn.o_proj": {"inputs": [[[0.5078125]]], "outputs": [[[0.6875]], [[1.328125]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.2.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.60546875]], [[5.96875]]]}, "language_model.model.layers.2.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.1640625]]]}, "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.60546875]]]}, "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.90625]]]}, "language_model.model.layers.2.self_attn.attn.impl.k_cache": {"inputs": [[[5.96875]]]}, "language_model.model.layers.2.self_attn.attn.impl.v_cache": {"inputs": [[[1.1640625]]]}, "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[7.375]], [[6.5625]], [[1.1640625]]], "outputs": [[[0.5078125]], [[1.0]]]}, "language_model.model.layers.2.feed_forward.gate_up_proj": {"inputs": [[[6.125]]], "params": {"weight": [[0.359375]]}}, "language_model.model.layers.2.feed_forward.down_proj": {"inputs": [[[5.75]]], "outputs": [[[4.75]], [[25.25]]], "params": {"weight": [[0.5234375]]}}, "language_model.model.layers.3.self_attn.qkv_proj": {"inputs": [[[7.21875]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.3.self_attn.o_proj": {"inputs": [[[0.88671875]]], "outputs": [[[0.59765625]], [[2.40625]]], "params": {"weight": [[0.453125]]}}, "language_model.model.layers.3.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.1201171875]], [[6.375]]]}, "language_model.model.layers.3.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[0.91796875]]]}, "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.1201171875]]]}, "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.80078125]]]}, "language_model.model.layers.3.self_attn.attn.impl.k_cache": {"inputs": [[[6.375]]]}, "language_model.model.layers.3.self_attn.attn.impl.v_cache": {"inputs": [[[0.91796875]]]}, "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[1.8125]], [[6.375]], [[0.91796875]]], "outputs": [[[0.88671875]], [[1.0]]]}, "language_model.model.layers.3.feed_forward.router": {"inputs": [[[8.75]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op": {"inputs": [[[8.75]]], "outputs": [[[235.0]], [[0.69140625]], [[9.9375]], [[14.8125]], [[7.40625]], [[0.0]], [[2.28125]], [[11.3125]], [[14.125]], [[10.875]], [[17.125]], [[6.3125]], [[7.8125]], [[8.875]], [[17.375]], [[7.09375]], [[14.875]], [[0.85546875]], [[3.546875]], [[5.78125]], [[11.9375]], [[6.625]], [[17.25]], [[15.1875]], [[16.5]], [[12.25]], [[2.015625]], [[11.6875]], [[15.25]], [[6.8125]], [[9.25]], [[16.0]], [[9.0625]], [[14.0625]], [[8.75]], [[13.4375]], [[12.0]], [[3.671875]], [[2.765625]], [[18.75]], [[4.6875]], [[9.5]], [[15.5]], [[6.3125]], [[5.96875]], [[16.5]], [[5.53125]], [[15.3125]], [[18.125]], [[2.3125]], [[15.9375]], [[15.75]], [[13.9375]], [[17.875]], [[3.765625]], [[10.625]], [[22.5]], [[11.3125]], [[17.625]], [[19.375]], [[7.40625]], [[6.09375]], [[17.625]], [[19.625]], [[0.1474609375]], [[15.6875]], [[19.25]], [[11.25]], [[0.2236328125]], [[3.5625]], [[0.0]], [[12.5]], [[13.0]], [[14.9375]], [[14.0]], [[1.5078125]], [[7.125]], [[11.5]], [[0.11181640625]], [[16.125]], [[15.625]], [[13.4375]], [[0.0]], [[3.015625]], [[37.75]], [[8.0]], [[272.0]], [[6.40625]], [[0.87109375]], [[19.0]], [[18.125]], [[14.8125]], [[6.0]], [[5.71875]], [[17.125]], [[0.0]], [[11.3125]], [[19.875]], [[10.1875]], [[7.25]], [[3.625]], [[17.125]], [[5.03125]], [[3.609375]], [[15.8125]], [[0.0]], [[7.1875]], [[10.875]], [[14.9375]], [[9.1875]], [[1.6640625]], [[12.0]], [[6.0]], [[17.125]], [[5.0625]], [[4.125]], [[1.3359375]], [[3.359375]], [[0.4609375]], [[8.875]], [[7.84375]], [[25.75]], [[11.0]], [[10.4375]], [[11.5]], [[3.78125]], [[15.625]], [[5.0]], [[10.0625]]]}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.75]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[8.75]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.3.feed_forward.shared_expert.down_proj": {"inputs": [[[8.8125]]], "outputs": [[[5.78125]], [[1.869226869072538e+35]]], "params": {"weight": [[0.6484375]]}}, "language_model.model.layers.4.self_attn.qkv_proj": {"inputs": [[[13.75]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.4.self_attn.o_proj": {"inputs": [[[1.671875]]], "outputs": [[[1.015625]], [[2.046875]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.4.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.80078125]], [[13.875]]]}, "language_model.model.layers.4.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.03125]]]}, "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.80078125]]]}, "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.6171875]]]}, "language_model.model.layers.4.self_attn.attn.impl.k_cache": {"inputs": [[[13.875]]]}, "language_model.model.layers.4.self_attn.attn.impl.v_cache": {"inputs": [[[2.03125]]]}, "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.375]], [[13.875]], [[2.03125]]], "outputs": [[[1.671875]], [[1.0]]]}, "language_model.model.layers.4.feed_forward.gate_up_proj": {"inputs": [[[7.09375]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.4.feed_forward.down_proj": {"inputs": [[[8.1875]]], "outputs": [[[2.796875]], [[4.96875]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.5.self_attn.qkv_proj": {"inputs": [[[17.375]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.5.self_attn.o_proj": {"inputs": [[[1.1796875]]], "outputs": [[[0.97265625]], [[2.21875]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.5.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.73046875]], [[12.1875]]]}, "language_model.model.layers.5.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.296875]]]}, "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.73046875]]]}, "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.59375]]]}, "language_model.model.layers.5.self_attn.attn.impl.k_cache": {"inputs": [[[12.1875]]]}, "language_model.model.layers.5.self_attn.attn.impl.v_cache": {"inputs": [[[2.296875]]]}, "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.8125]], [[12.1875]], [[2.296875]]], "outputs": [[[1.1796875]], [[1.0]]]}, "language_model.model.layers.5.feed_forward.router": {"inputs": [[[19.25]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op": {"inputs": [[[19.25]]], "outputs": [[[438.0]], [[4.71875]], [[0.87890625]], [[7.40625]], [[17.75]], [[12.5]], [[5.65625]], [[9.625]], [[12.375]], [[13.4375]], [[19.25]], [[9.125]], [[26.25]], [[25.5]], [[6.125]], [[13.6875]], [[28.375]], [[13.0625]], [[16.125]], [[25.25]], [[8.625]], [[16.125]], [[5.0625]], [[8.6875]], [[21.0]], [[25.0]], [[5.125]], [[9.5]], [[190.0]], [[7.75]], [[20.75]], [[17.75]], [[29.0]], [[22.25]], [[23.125]], [[17.0]], [[14.5]], [[9.4375]], [[5.8125]], [[14.4375]], [[18.75]], [[9.625]], [[4.59375]], [[8.0]], [[12.75]], [[11.25]], [[14.5]], [[9.4375]], [[10.75]], [[5.6875]], [[14.0]], [[4.34375]], [[11.375]], [[0.9453125]], [[4.34375]], [[11.8125]], [[4.46875]], [[14.5625]], [[19.0]], [[14.25]], [[1.140625]], [[6.78125]], [[13.75]], [[8.125]], [[18.5]], [[8.5625]], [[12.75]], [[8.6875]], [[7.1875]], [[17.625]], [[5.1875]], [[5.53125]], [[16.375]], [[26.125]], [[17.5]], [[6.125]], [[9.3125]], [[18.875]], [[14.25]], [[15.8125]], [[17.875]], [[5.0625]], [[22.0]], [[7.90625]], [[12.3125]], [[4.125]], [[12.0]], [[0.1533203125]], [[30.5]], [[15.9375]], [[20.375]], [[18.875]], [[7.78125]], [[22.625]], [[13.25]], [[20.125]], [[2.296875]], [[19.875]], [[6.46875]], [[4.71875]], [[17.375]], [[6.9375]], [[3.203125]], [[1.3671875]], [[20.75]], [[19.75]], [[11.4375]], [[8.9375]], [[22.375]], [[9.4375]], [[21.125]], [[6.5]], [[5.65625]], [[15.375]], [[3.671875]], [[18.375]], [[12.75]], [[2.046875]], [[5.34375]], [[1.1875]], [[9.9375]], [[14.3125]], [[14.0]], [[12.0]], [[15.75]], [[17.625]], [[14.1875]], [[7.75]], [[7.53125]]]}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.392578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.07568359375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.5625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.78515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.6640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.59375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.384765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.357421875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[19.25]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.5.feed_forward.shared_expert.down_proj": {"inputs": [[[6.125]]], "outputs": [[[0.671875]], [[1.188422437713965e+29]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.6.self_attn.qkv_proj": {"inputs": [[[12.9375]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.6.self_attn.o_proj": {"inputs": [[[1.78125]]], "outputs": [[[0.66796875]], [[1.5234375]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.6.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.76953125]], [[10.25]]]}, "language_model.model.layers.6.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.515625]]]}, "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.76953125]]]}, "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[1.796875]]]}, "language_model.model.layers.6.self_attn.attn.impl.k_cache": {"inputs": [[[10.25]]]}, "language_model.model.layers.6.self_attn.attn.impl.v_cache": {"inputs": [[[2.515625]]]}, "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.0625]], [[10.25]], [[2.515625]]], "outputs": [[[1.78125]], [[1.0]]]}, "language_model.model.layers.6.feed_forward.gate_up_proj": {"inputs": [[[7.71875]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.6.feed_forward.down_proj": {"inputs": [[[7.65625]]], "outputs": [[[2.328125]], [[11.1875]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.7.self_attn.qkv_proj": {"inputs": [[[7.09375]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.7.self_attn.o_proj": {"inputs": [[[0.515625]]], "outputs": [[[1.84375]], [[3.84375]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.7.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.359375]], [[5.84375]]]}, "language_model.model.layers.7.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[1.2265625]]]}, "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.359375]]]}, "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[0.875]]]}, "language_model.model.layers.7.self_attn.attn.impl.k_cache": {"inputs": [[[5.84375]]]}, "language_model.model.layers.7.self_attn.attn.impl.v_cache": {"inputs": [[[1.2265625]]]}, "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[4.5]], [[5.84375]], [[1.2265625]]], "outputs": [[[0.515625]], [[1.0]]]}, "language_model.model.layers.7.feed_forward.router": {"inputs": [[[11.6875]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op": {"inputs": [[[11.6875]]], "outputs": [[[3.421875]], [[14.5625]], [[16.875]], [[23.375]], [[13.25]], [[3.765625]], [[14.0625]], [[12.375]], [[15.6875]], [[9.0625]], [[19.375]], [[14.4375]], [[15.25]], [[14.5625]], [[20.25]], [[2.078125]], [[5.8125]], [[13.625]], [[8.5]], [[21.0]], [[18.625]], [[15.4375]], [[26.0]], [[17.5]], [[26.875]], [[19.25]], [[9.3125]], [[8.75]], [[9.0625]], [[13.5]], [[8.75]], [[9.8125]], [[26.0]], [[7.25]], [[11.6875]], [[6.96875]], [[7.3125]], [[7.65625]], [[15.125]], [[22.875]], [[15.3125]], [[6.0]], [[5.5]], [[1.3046875]], [[24.5]], [[8.5625]], [[25.0]], [[0.8515625]], [[27.0]], [[17.25]], [[6.84375]], [[14.8125]], [[25.125]], [[20.625]], [[14.125]], [[27.125]], [[30.375]], [[8.9375]], [[2.65625]], [[3.859375]], [[9.1875]], [[6.59375]], [[18.125]], [[3.1875]], [[10.0625]], [[20.25]], [[15.9375]], [[17.75]], [[8.5]], [[6.1875]], [[5.09375]], [[31.125]], [[4.75]], [[11.6875]], [[15.25]], [[38.75]], [[4.09375]], [[5.3125]], [[9.25]], [[1.375]], [[9.5625]], [[15.9375]], [[11.1875]], [[2.140625]], [[22.375]], [[8.375]], [[14.3125]], [[1.4921875]], [[21.875]], [[5.03125]], [[26.25]], [[36.5]], [[20.5]], [[8.8125]], [[3.671875]], [[20.375]], [[6.375]], [[17.0]], [[5.75]], [[21.375]], [[8.5625]], [[14.6875]], [[17.125]], [[4.5625]], [[0.59375]], [[45.75]], [[34.25]], [[22.25]], [[9.75]], [[0.79296875]], [[26.25]], [[18.125]], [[26.75]], [[12.4375]], [[37.5]], [[19.875]], [[5.03125]], [[8.25]], [[15.3125]], [[6.6875]], [[17.25]], [[22.875]], [[29.375]], [[26.25]], [[3.484375]], [[4.25]], [[1.8359375]], [[52.25]], [[20.75]]]}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.53125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.61328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[11.6875]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.7.feed_forward.shared_expert.down_proj": {"inputs": [[[9.125]]], "outputs": [[[0.625]], [[1.869226869072538e+35]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.8.self_attn.qkv_proj": {"inputs": [[[15.125]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.8.self_attn.o_proj": {"inputs": [[[2.21875]]], "outputs": [[[1.9375]], [[2.109375]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.8.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.81640625]], [[8.1875]]]}, "language_model.model.layers.8.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.96875]]]}, "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.81640625]]]}, "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.75]]]}, "language_model.model.layers.8.self_attn.attn.impl.k_cache": {"inputs": [[[8.1875]]]}, "language_model.model.layers.8.self_attn.attn.impl.v_cache": {"inputs": [[[3.96875]]]}, "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.0]], [[8.1875]], [[3.96875]]], "outputs": [[[2.21875]], [[1.0]]]}, "language_model.model.layers.8.feed_forward.gate_up_proj": {"inputs": [[[8.25]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.8.feed_forward.down_proj": {"inputs": [[[9.125]]], "outputs": [[[2.5625]], [[9.25]]], "params": {"weight": [[0.703125]]}}, "language_model.model.layers.9.self_attn.qkv_proj": {"inputs": [[[15.75]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.9.self_attn.o_proj": {"inputs": [[[3.15625]]], "outputs": [[[1.3125]], [[2.875]]], "params": {"weight": [[0.302734375]]}}, "language_model.model.layers.9.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.73046875]], [[8.5]]]}, "language_model.model.layers.9.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.28125]]]}, "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.73046875]]]}, "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.15625]]]}, "language_model.model.layers.9.self_attn.attn.impl.k_cache": {"inputs": [[[8.5]]]}, "language_model.model.layers.9.self_attn.attn.impl.v_cache": {"inputs": [[[4.28125]]]}, "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.1875]], [[8.5]], [[4.28125]]], "outputs": [[[3.03125]], [[1.0]]]}, "language_model.model.layers.9.feed_forward.router": {"inputs": [[[11.75]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op": {"inputs": [[[11.75]]], "outputs": [[[3.453125]], [[12.5625]], [[24.25]], [[4.71875]], [[33.75]], [[32.25]], [[20.375]], [[25.75]], [[27.25]], [[26.875]], [[20.375]], [[34.25]], [[5.96875]], [[39.5]], [[31.25]], [[10.5]], [[5.78125]], [[10.5]], [[20.5]], [[23.25]], [[20.375]], [[11.4375]], [[12.375]], [[15.3125]], [[22.375]], [[14.6875]], [[15.0625]], [[40.25]], [[26.125]], [[15.8125]], [[38.0]], [[24.875]], [[14.0]], [[35.0]], [[37.0]], [[35.0]], [[6.90625]], [[28.125]], [[25.75]], [[7.3125]], [[17.375]], [[7.6875]], [[8.9375]], [[19.5]], [[29.0]], [[7.5625]], [[16.0]], [[32.25]], [[18.125]], [[14.625]], [[30.75]], [[9.9375]], [[23.375]], [[13.1875]], [[15.8125]], [[19.375]], [[12.75]], [[43.0]], [[17.625]], [[41.75]], [[21.125]], [[15.375]], [[14.625]], [[24.25]], [[43.75]], [[7.40625]], [[26.375]], [[34.25]], [[22.875]], [[3.9375]], [[18.125]], [[22.25]], [[32.75]], [[9.625]], [[13.0]], [[4.0625]], [[29.875]], [[35.75]], [[28.125]], [[3.109375]], [[14.5625]], [[12.4375]], [[36.5]], [[25.375]], [[61.0]], [[17.875]], [[30.125]], [[14.4375]], [[12.0625]], [[19.75]], [[49.0]], [[5.84375]], [[12.1875]], [[12.6875]], [[12.9375]], [[34.75]], [[35.75]], [[15.875]], [[7.90625]], [[16.375]], [[13.0625]], [[26.0]], [[19.5]], [[30.375]], [[9.0]], [[32.25]], [[11.25]], [[28.5]], [[26.25]], [[21.75]], [[15.8125]], [[17.125]], [[3.6875]], [[21.75]], [[5.28125]], [[50.25]], [[29.625]], [[6.84375]], [[7.03125]], [[11.75]], [[8.25]], [[32.25]], [[15.9375]], [[23.25]], [[43.0]], [[52.75]], [[41.5]], [[12.3125]], [[28.75]]]}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[11.75]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.9.feed_forward.shared_expert.down_proj": {"inputs": [[[13.6875]]], "outputs": [[[0.74609375]], [[1.895188353365212e+35]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.10.self_attn.qkv_proj": {"inputs": [[[16.125]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.10.self_attn.o_proj": {"inputs": [[[2.453125]]], "outputs": [[[1.6015625]], [[3.09375]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.10.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.89453125]], [[17.25]]]}, "language_model.model.layers.10.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[2.78125]]]}, "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.89453125]]]}, "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.03125]]]}, "language_model.model.layers.10.self_attn.attn.impl.k_cache": {"inputs": [[[17.25]]]}, "language_model.model.layers.10.self_attn.attn.impl.v_cache": {"inputs": [[[2.78125]]]}, "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.1875]], [[17.25]], [[2.78125]]], "outputs": [[[2.453125]], [[1.0]]]}, "language_model.model.layers.10.feed_forward.gate_up_proj": {"inputs": [[[9.875]]], "params": {"weight": [[0.310546875]]}}, "language_model.model.layers.10.feed_forward.down_proj": {"inputs": [[[10.375]]], "outputs": [[[6.90625]], [[11.3125]]], "params": {"weight": [[0.67578125]]}}, "language_model.model.layers.11.self_attn.qkv_proj": {"inputs": [[[13.5625]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.11.self_attn.o_proj": {"inputs": [[[2.765625]]], "outputs": [[[8.0625]], [[7.875]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.11.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.87109375]], [[8.9375]]]}, "language_model.model.layers.11.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.609375]]]}, "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.87109375]]]}, "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.765625]]]}, "language_model.model.layers.11.self_attn.attn.impl.k_cache": {"inputs": [[[8.9375]]]}, "language_model.model.layers.11.self_attn.attn.impl.v_cache": {"inputs": [[[3.609375]]]}, "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.375]], [[8.9375]], [[3.609375]]], "outputs": [[[2.453125]], [[1.0]]]}, "language_model.model.layers.11.feed_forward.router": {"inputs": [[[13.375]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op": {"inputs": [[[13.375]]], "outputs": [[[3.859375]], [[17.5]], [[13.3125]], [[29.875]], [[15.875]], [[22.75]], [[24.25]], [[1.21875]], [[18.5]], [[15.375]], [[7.96875]], [[36.0]], [[13.1875]], [[28.625]], [[43.0]], [[20.5]], [[43.0]], [[32.25]], [[32.75]], [[28.25]], [[23.875]], [[12.25]], [[46.25]], [[40.5]], [[22.75]], [[15.0]], [[30.75]], [[18.75]], [[19.75]], [[8.125]], [[41.25]], [[25.375]], [[8.0]], [[16.75]], [[13.5625]], [[31.5]], [[56.5]], [[21.25]], [[20.25]], [[36.0]], [[25.875]], [[4.75]], [[15.75]], [[9.875]], [[31.75]], [[7.90625]], [[2.09375]], [[16.75]], [[37.25]], [[23.0]], [[7.75]], [[18.375]], [[25.75]], [[7.21875]], [[42.0]], [[64.5]], [[39.25]], [[14.625]], [[7.5]], [[38.0]], [[24.375]], [[18.875]], [[10.8125]], [[18.875]], [[23.25]], [[9.125]], [[42.25]], [[22.375]], [[45.5]], [[19.5]], [[24.375]], [[9.5625]], [[12.8125]], [[1.765625]], [[23.75]], [[15.875]], [[24.0]], [[23.125]], [[4.4375]], [[60.5]], [[29.75]], [[15.8125]], [[13.6875]], [[30.375]], [[11.5]], [[2.640625]], [[1.4921875]], [[36.75]], [[9.375]], [[8.9375]], [[23.125]], [[32.25]], [[18.125]], [[24.75]], [[36.75]], [[14.9375]], [[10.75]], [[29.0]], [[24.625]], [[13.8125]], [[27.125]], [[6.21875]], [[46.5]], [[22.125]], [[10.1875]], [[41.5]], [[23.625]], [[14.5]], [[9.25]], [[13.375]], [[15.6875]], [[34.0]], [[12.1875]], [[5.6875]], [[34.25]], [[31.875]], [[52.0]], [[49.0]], [[33.75]], [[26.125]], [[29.5]], [[39.75]], [[22.25]], [[15.9375]], [[13.625]], [[9.375]], [[12.875]], [[45.25]], [[18.0]]]}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.380859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.07666015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0771484375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.076171875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[13.375]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.11.feed_forward.shared_expert.down_proj": {"inputs": [[[13.375]]], "outputs": [[[1.578125]], [[1.895188353365212e+35]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.12.self_attn.qkv_proj": {"inputs": [[[18.625]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.12.self_attn.o_proj": {"inputs": [[[3.546875]]], "outputs": [[[1.3359375]], [[3.640625]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.12.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.8125]], [[14.9375]]]}, "language_model.model.layers.12.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.59375]]]}, "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.8125]]]}, "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.84375]]]}, "language_model.model.layers.12.self_attn.attn.impl.k_cache": {"inputs": [[[14.9375]]]}, "language_model.model.layers.12.self_attn.attn.impl.v_cache": {"inputs": [[[3.59375]]]}, "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[9.375]], [[14.9375]], [[3.59375]]], "outputs": [[[3.546875]], [[1.0]]]}, "language_model.model.layers.12.feed_forward.gate_up_proj": {"inputs": [[[10.75]]], "params": {"weight": [[0.322265625]]}}, "language_model.model.layers.12.feed_forward.down_proj": {"inputs": [[[14.1875]]], "outputs": [[[9.0]], [[17.375]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.13.self_attn.qkv_proj": {"inputs": [[[20.875]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.13.self_attn.o_proj": {"inputs": [[[2.40625]]], "outputs": [[[1.6875]], [[6.9375]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.13.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.90234375]], [[16.0]]]}, "language_model.model.layers.13.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.015625]]]}, "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.90234375]]]}, "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.21875]]]}, "language_model.model.layers.13.self_attn.attn.impl.k_cache": {"inputs": [[[16.0]]]}, "language_model.model.layers.13.self_attn.attn.impl.v_cache": {"inputs": [[[3.015625]]]}, "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.8125]], [[16.0]], [[3.015625]]], "outputs": [[[2.40625]], [[1.0]]]}, "language_model.model.layers.13.feed_forward.router": {"inputs": [[[17.625]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op": {"inputs": [[[17.625]]], "outputs": [[[3.9375]], [[22.25]], [[38.0]], [[22.5]], [[50.75]], [[63.0]], [[13.25]], [[14.75]], [[11.75]], [[28.25]], [[4.8125]], [[38.25]], [[55.25]], [[9.6875]], [[26.125]], [[21.25]], [[22.5]], [[10.875]], [[20.75]], [[46.5]], [[24.5]], [[18.0]], [[40.25]], [[11.4375]], [[12.1875]], [[38.5]], [[32.5]], [[9.8125]], [[44.5]], [[31.875]], [[10.4375]], [[17.125]], [[19.25]], [[71.0]], [[27.75]], [[31.875]], [[27.25]], [[48.0]], [[43.75]], [[32.25]], [[8.8125]], [[12.0625]], [[30.0]], [[34.0]], [[13.25]], [[21.25]], [[30.75]], [[22.25]], [[18.375]], [[13.6875]], [[54.75]], [[31.5]], [[33.0]], [[13.9375]], [[64.5]], [[5.6875]], [[18.375]], [[39.75]], [[20.25]], [[25.0]], [[26.25]], [[9.75]], [[26.125]], [[18.75]], [[30.375]], [[9.4375]], [[45.75]], [[7.71875]], [[38.0]], [[23.375]], [[12.0]], [[45.5]], [[54.5]], [[34.25]], [[13.0]], [[22.375]], [[23.875]], [[49.5]], [[30.875]], [[34.5]], [[25.0]], [[10.0]], [[49.0]], [[5.15625]], [[33.0]], [[49.75]], [[34.0]], [[26.125]], [[33.5]], [[2.484375]], [[38.0]], [[36.5]], [[61.25]], [[2.3125]], [[44.0]], [[33.75]], [[14.3125]], [[14.3125]], [[12.75]], [[27.125]], [[59.75]], [[6.40625]], [[2.046875]], [[25.125]], [[32.0]], [[34.25]], [[48.25]], [[52.75]], [[38.25]], [[18.875]], [[36.0]], [[20.625]], [[52.5]], [[16.125]], [[31.25]], [[33.0]], [[32.75]], [[17.5]], [[29.375]], [[46.0]], [[19.375]], [[21.375]], [[24.25]], [[23.625]], [[12.5625]], [[29.625]], [[24.625]], [[10.0]], [[53.5]]]}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.5234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[17.625]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.13.feed_forward.shared_expert.down_proj": {"inputs": [[[14.875]]], "outputs": [[[3.21875]], [[1.6096120261457966e+35]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.14.self_attn.qkv_proj": {"inputs": [[[18.75]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.14.self_attn.o_proj": {"inputs": [[[3.390625]]], "outputs": [[[2.140625]], [[4.875]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.14.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.734375]], [[11.125]]]}, "language_model.model.layers.14.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.34375]]]}, "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.734375]]]}, "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.390625]]]}, "language_model.model.layers.14.self_attn.attn.impl.k_cache": {"inputs": [[[11.125]]]}, "language_model.model.layers.14.self_attn.attn.impl.v_cache": {"inputs": [[[4.34375]]]}, "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.9375]], [[11.125]], [[4.34375]]], "outputs": [[[3.390625]], [[1.0]]]}, "language_model.model.layers.14.feed_forward.gate_up_proj": {"inputs": [[[12.0625]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.14.feed_forward.down_proj": {"inputs": [[[15.75]]], "outputs": [[[4.65625]], [[28.25]]], "params": {"weight": [[0.609375]]}}, "language_model.model.layers.15.self_attn.qkv_proj": {"inputs": [[[20.875]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.15.self_attn.o_proj": {"inputs": [[[3.71875]]], "outputs": [[[21.875]], [[23.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.15.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.5625]], [[9.3125]]]}, "language_model.model.layers.15.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.4375]]]}, "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.5625]]]}, "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.4375]]]}, "language_model.model.layers.15.self_attn.attn.impl.k_cache": {"inputs": [[[9.3125]]]}, "language_model.model.layers.15.self_attn.attn.impl.v_cache": {"inputs": [[[4.4375]]]}, "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[8.0625]], [[9.3125]], [[4.4375]]], "outputs": [[[3.71875]], [[1.0]]]}, "language_model.model.layers.15.feed_forward.router": {"inputs": [[[19.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op": {"inputs": [[[19.0]]], "outputs": [[[4.6875]], [[26.75]], [[21.5]], [[24.875]], [[25.5]], [[14.75]], [[60.0]], [[22.75]], [[61.25]], [[44.25]], [[54.5]], [[17.125]], [[19.0]], [[19.25]], [[25.0]], [[4.25]], [[15.875]], [[47.0]], [[51.0]], [[16.125]], [[12.9375]], [[28.375]], [[20.0]], [[47.0]], [[39.5]], [[20.875]], [[9.5625]], [[41.75]], [[21.0]], [[63.5]], [[38.0]], [[42.75]], [[23.0]], [[14.5]], [[10.8125]], [[35.25]], [[33.0]], [[6.65625]], [[36.0]], [[51.5]], [[59.75]], [[40.25]], [[63.5]], [[59.5]], [[42.75]], [[47.5]], [[56.25]], [[13.625]], [[39.5]], [[14.75]], [[61.5]], [[28.875]], [[49.0]], [[35.0]], [[42.5]], [[36.75]], [[24.25]], [[27.125]], [[63.0]], [[23.375]], [[33.0]], [[18.875]], [[46.5]], [[26.875]], [[32.75]], [[16.25]], [[39.75]], [[34.0]], [[56.0]], [[13.5625]], [[27.625]], [[44.5]], [[16.375]], [[20.25]], [[38.5]], [[43.75]], [[39.75]], [[15.9375]], [[44.5]], [[21.25]], [[7.9375]], [[39.0]], [[47.75]], [[14.9375]], [[13.0]], [[13.625]], [[23.375]], [[24.25]], [[12.75]], [[23.75]], [[41.75]], [[3.1875]], [[13.375]], [[18.75]], [[31.375]], [[38.25]], [[54.25]], [[39.25]], [[26.625]], [[18.125]], [[73.5]], [[26.375]], [[67.5]], [[35.25]], [[20.125]], [[43.5]], [[25.625]], [[10.5]], [[39.25]], [[50.25]], [[16.625]], [[44.5]], [[46.0]], [[48.25]], [[3.921875]], [[21.875]], [[17.75]], [[32.5]], [[7.0]], [[33.5]], [[22.0]], [[50.5]], [[34.0]], [[21.75]], [[31.25]], [[85.5]], [[13.1875]], [[11.4375]], [[39.0]]]}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[19.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.15.feed_forward.shared_expert.down_proj": {"inputs": [[[20.625]]], "outputs": [[[2.234375]], [[1.188422437713965e+29]]], "params": {"weight": [[0.38671875]]}}, "language_model.model.layers.16.self_attn.qkv_proj": {"inputs": [[[20.375]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.16.self_attn.o_proj": {"inputs": [[[3.3125]]], "outputs": [[[2.75]], [[6.28125]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.16.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.03125]], [[17.375]]]}, "language_model.model.layers.16.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.90625]]]}, "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.03125]]]}, "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.875]]]}, "language_model.model.layers.16.self_attn.attn.impl.k_cache": {"inputs": [[[17.375]]]}, "language_model.model.layers.16.self_attn.attn.impl.v_cache": {"inputs": [[[3.90625]]]}, "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.0625]], [[17.375]], [[3.875]]], "outputs": [[[3.3125]], [[1.0]]]}, "language_model.model.layers.16.feed_forward.gate_up_proj": {"inputs": [[[14.8125]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.16.feed_forward.down_proj": {"inputs": [[[17.875]]], "outputs": [[[7.03125]], [[22.625]]], "params": {"weight": [[0.515625]]}}, "language_model.model.layers.17.self_attn.qkv_proj": {"inputs": [[[19.875]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.17.self_attn.o_proj": {"inputs": [[[3.1875]]], "outputs": [[[17.5]], [[20.75]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.17.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.890625]], [[16.0]]]}, "language_model.model.layers.17.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[3.90625]]]}, "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.890625]]]}, "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[2.78125]]]}, "language_model.model.layers.17.self_attn.attn.impl.k_cache": {"inputs": [[[16.0]]]}, "language_model.model.layers.17.self_attn.attn.impl.v_cache": {"inputs": [[[3.9375]]]}, "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.125]], [[16.0]], [[3.9375]]], "outputs": [[[3.1875]], [[1.0]]]}, "language_model.model.layers.17.feed_forward.router": {"inputs": [[[23.875]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op": {"inputs": [[[23.875]]], "outputs": [[[35.5]], [[23.125]], [[16.5]], [[50.25]], [[8.0]], [[39.25]], [[28.875]], [[57.0]], [[18.25]], [[58.75]], [[6.03125]], [[12.0625]], [[31.125]], [[26.5]], [[25.0]], [[26.0]], [[52.5]], [[33.0]], [[37.0]], [[15.5625]], [[26.625]], [[25.375]], [[46.0]], [[63.25]], [[58.75]], [[12.0625]], [[21.375]], [[25.875]], [[15.25]], [[66.0]], [[55.0]], [[30.375]], [[15.9375]], [[17.625]], [[13.8125]], [[24.125]], [[86.0]], [[24.875]], [[13.625]], [[19.0]], [[72.0]], [[43.5]], [[42.5]], [[13.3125]], [[20.0]], [[32.25]], [[79.0]], [[23.25]], [[63.5]], [[36.0]], [[31.125]], [[72.0]], [[63.75]], [[13.5]], [[12.5]], [[23.125]], [[3.734375]], [[3.828125]], [[48.0]], [[13.125]], [[17.0]], [[50.75]], [[24.375]], [[11.9375]], [[16.375]], [[36.5]], [[45.25]], [[13.9375]], [[35.25]], [[65.0]], [[12.0625]], [[29.625]], [[7.625]], [[58.75]], [[81.5]], [[31.0]], [[37.25]], [[39.25]], [[67.0]], [[8.25]], [[62.5]], [[13.6875]], [[26.75]], [[67.5]], [[29.875]], [[42.5]], [[40.25]], [[12.9375]], [[17.875]], [[32.75]], [[38.5]], [[44.0]], [[31.5]], [[21.125]], [[28.25]], [[8.3125]], [[28.25]], [[8.3125]], [[3.5]], [[38.0]], [[20.0]], [[12.25]], [[35.75]], [[32.5]], [[26.875]], [[46.25]], [[18.875]], [[28.625]], [[41.0]], [[19.75]], [[21.0]], [[40.75]], [[24.125]], [[62.25]], [[20.875]], [[18.25]], [[14.3125]], [[32.75]], [[49.25]], [[23.625]], [[50.5]], [[28.875]], [[46.25]], [[25.75]], [[13.0]], [[17.375]], [[11.625]], [[8.3125]], [[45.0]]]}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.6171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.341796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.35546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.875]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.17.feed_forward.shared_expert.down_proj": {"inputs": [[[26.625]]], "outputs": [[[13.1875]], [[1.5382179443409427e+35]]], "params": {"weight": [[0.81640625]]}}, "language_model.model.layers.18.self_attn.qkv_proj": {"inputs": [[[22.125]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.18.self_attn.o_proj": {"inputs": [[[4.25]]], "outputs": [[[6.375]], [[11.5625]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.18.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.921875]], [[17.625]]]}, "language_model.model.layers.18.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[4.34375]]]}, "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.921875]]]}, "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.578125]]]}, "language_model.model.layers.18.self_attn.attn.impl.k_cache": {"inputs": [[[17.625]]]}, "language_model.model.layers.18.self_attn.attn.impl.v_cache": {"inputs": [[[4.34375]]]}, "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.3125]], [[17.625]], [[4.34375]]], "outputs": [[[4.25]], [[1.0]]]}, "language_model.model.layers.18.feed_forward.gate_up_proj": {"inputs": [[[18.5]]], "params": {"weight": [[0.3671875]]}}, "language_model.model.layers.18.feed_forward.down_proj": {"inputs": [[[18.625]]], "outputs": [[[8.25]], [[358.0]]], "params": {"weight": [[0.5546875]]}}, "language_model.model.layers.19.self_attn.qkv_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.19.self_attn.o_proj": {"inputs": [[[6.15625]]], "outputs": [[[31.625]], [[36.75]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.19.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.984375]], [[13.6875]]]}, "language_model.model.layers.19.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.6875]]]}, "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.984375]]]}, "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.59375]]]}, "language_model.model.layers.19.self_attn.attn.impl.k_cache": {"inputs": [[[13.6875]]]}, "language_model.model.layers.19.self_attn.attn.impl.v_cache": {"inputs": [[[6.6875]]]}, "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.1875]], [[13.6875]], [[6.6875]]], "outputs": [[[6.15625]], [[1.0]]]}, "language_model.model.layers.19.feed_forward.router": {"inputs": [[[23.25]]], "params": {"weight": [[0.3515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op": {"inputs": [[[23.25]]], "outputs": [[[12.3125]], [[38.5]], [[56.25]], [[16.5]], [[35.25]], [[36.5]], [[10.75]], [[15.75]], [[66.5]], [[13.75]], [[17.875]], [[7.40625]], [[52.75]], [[35.75]], [[36.25]], [[38.25]], [[9.625]], [[21.625]], [[27.75]], [[47.75]], [[18.25]], [[24.875]], [[28.75]], [[52.5]], [[29.0]], [[48.0]], [[67.5]], [[19.5]], [[51.25]], [[57.0]], [[30.875]], [[11.375]], [[26.375]], [[41.5]], [[19.875]], [[37.0]], [[53.75]], [[10.75]], [[42.75]], [[39.25]], [[22.5]], [[53.5]], [[15.6875]], [[37.25]], [[57.5]], [[25.0]], [[9.5625]], [[14.125]], [[15.1875]], [[23.375]], [[60.75]], [[20.375]], [[49.25]], [[7.59375]], [[21.0]], [[37.25]], [[18.625]], [[20.5]], [[59.0]], [[49.75]], [[32.0]], [[18.375]], [[30.0]], [[9.5]], [[19.875]], [[48.0]], [[35.25]], [[23.375]], [[31.75]], [[37.75]], [[15.0]], [[23.875]], [[40.25]], [[36.5]], [[33.0]], [[35.0]], [[16.75]], [[13.3125]], [[22.875]], [[42.5]], [[8.8125]], [[42.0]], [[19.125]], [[28.375]], [[41.0]], [[28.875]], [[38.25]], [[36.25]], [[6.875]], [[38.0]], [[32.0]], [[19.375]], [[31.0]], [[17.375]], [[28.0]], [[17.375]], [[15.9375]], [[19.875]], [[40.5]], [[29.875]], [[8.875]], [[64.0]], [[25.0]], [[24.25]], [[31.625]], [[24.875]], [[25.0]], [[48.75]], [[28.5]], [[40.0]], [[29.875]], [[21.25]], [[30.375]], [[51.5]], [[43.75]], [[18.125]], [[16.125]], [[27.0]], [[32.75]], [[26.0]], [[39.75]], [[14.25]], [[18.0]], [[18.875]], [[44.75]], [[7.84375]], [[22.0]], [[27.625]], [[32.0]]]}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.65625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.44140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.41796875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.5625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0791015625]]}}, "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.25]]], "params": {"weight": [[0.404296875]]}}, "language_model.model.layers.19.feed_forward.shared_expert.down_proj": {"inputs": [[[31.75]]], "outputs": [[[31.875]], [[1.188422437713965e+29]]], "params": {"weight": [[1.015625]]}}, "language_model.model.layers.20.self_attn.qkv_proj": {"inputs": [[[52.5]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.20.self_attn.o_proj": {"inputs": [[[4.90625]]], "outputs": [[[9.5]], [[18.375]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.20.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.98828125]], [[12.75]]]}, "language_model.model.layers.20.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.5625]]]}, "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.98828125]]]}, "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.84375]]]}, "language_model.model.layers.20.self_attn.attn.impl.k_cache": {"inputs": [[[12.75]]]}, "language_model.model.layers.20.self_attn.attn.impl.v_cache": {"inputs": [[[8.5625]]]}, "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.5625]], [[12.75]], [[8.5625]]], "outputs": [[[4.90625]], [[1.0]]]}, "language_model.model.layers.20.feed_forward.gate_up_proj": {"inputs": [[[41.5]]], "params": {"weight": [[0.28515625]]}}, "language_model.model.layers.20.feed_forward.down_proj": {"inputs": [[[19.5]]], "outputs": [[[9.4375]], [[38.5]]], "params": {"weight": [[0.609375]]}}, "language_model.model.layers.21.self_attn.qkv_proj": {"inputs": [[[56.25]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.21.self_attn.o_proj": {"inputs": [[[5.84375]]], "outputs": [[[25.125]], [[37.75]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.21.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.015625]], [[17.375]]]}, "language_model.model.layers.21.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.8125]]]}, "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.015625]]]}, "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.625]]]}, "language_model.model.layers.21.self_attn.attn.impl.k_cache": {"inputs": [[[17.375]]]}, "language_model.model.layers.21.self_attn.attn.impl.v_cache": {"inputs": [[[6.8125]]]}, "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.875]], [[17.375]], [[6.8125]]], "outputs": [[[5.84375]], [[1.0]]]}, "language_model.model.layers.21.feed_forward.router": {"inputs": [[[24.5]]], "params": {"weight": [[0.45703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op": {"inputs": [[[24.5]]], "outputs": [[[4.4375]], [[20.125]], [[53.0]], [[38.25]], [[30.75]], [[9.1875]], [[36.0]], [[18.125]], [[34.0]], [[22.5]], [[28.0]], [[11.375]], [[9.6875]], [[11.375]], [[65.0]], [[17.625]], [[40.0]], [[26.75]], [[38.0]], [[37.5]], [[36.0]], [[21.125]], [[45.75]], [[19.625]], [[16.125]], [[29.75]], [[20.375]], [[19.125]], [[13.5625]], [[45.25]], [[32.75]], [[8.875]], [[38.25]], [[46.25]], [[18.75]], [[30.5]], [[12.9375]], [[26.5]], [[58.25]], [[29.5]], [[45.75]], [[10.1875]], [[20.625]], [[20.375]], [[9.125]], [[42.75]], [[21.625]], [[14.5]], [[52.0]], [[23.0]], [[21.375]], [[61.75]], [[76.5]], [[37.75]], [[17.25]], [[10.1875]], [[25.0]], [[36.0]], [[15.4375]], [[17.875]], [[46.0]], [[26.625]], [[16.75]], [[28.25]], [[61.0]], [[13.6875]], [[66.5]], [[23.125]], [[19.75]], [[13.8125]], [[9.4375]], [[22.75]], [[21.625]], [[29.25]], [[34.75]], [[15.6875]], [[11.5]], [[42.5]], [[11.875]], [[25.75]], [[17.75]], [[27.625]], [[22.375]], [[62.25]], [[21.75]], [[24.375]], [[31.25]], [[7.09375]], [[21.75]], [[9.1875]], [[18.75]], [[25.875]], [[8.25]], [[44.75]], [[10.5]], [[16.875]], [[17.0]], [[25.25]], [[23.875]], [[23.75]], [[29.875]], [[11.8125]], [[21.25]], [[24.125]], [[51.0]], [[25.875]], [[23.625]], [[35.25]], [[44.75]], [[61.25]], [[22.125]], [[49.5]], [[20.375]], [[17.875]], [[18.0]], [[33.5]], [[20.625]], [[22.625]], [[12.8125]], [[25.375]], [[50.0]], [[18.125]], [[28.875]], [[23.25]], [[27.625]], [[14.0]], [[35.5]], [[16.0]], [[45.25]]]}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.39453125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.43359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.341796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.078125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.07763671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[24.5]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.21.feed_forward.shared_expert.down_proj": {"inputs": [[[21.375]]], "outputs": [[[9.6875]], [[1.188422437713965e+29]]], "params": {"weight": [[0.80078125]]}}, "language_model.model.layers.22.self_attn.qkv_proj": {"inputs": [[[55.75]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.22.self_attn.o_proj": {"inputs": [[[5.90625]]], "outputs": [[[17.25]], [[23.25]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.22.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.95703125]], [[17.75]]]}, "language_model.model.layers.22.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.625]]]}, "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.95703125]]]}, "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.90625]]]}, "language_model.model.layers.22.self_attn.attn.impl.k_cache": {"inputs": [[[18.625]]]}, "language_model.model.layers.22.self_attn.attn.impl.v_cache": {"inputs": [[[6.625]]]}, "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.0]], [[18.625]], [[6.625]]], "outputs": [[[5.90625]], [[1.0]]]}, "language_model.model.layers.22.feed_forward.gate_up_proj": {"inputs": [[[44.75]]], "params": {"weight": [[0.404296875]]}}, "language_model.model.layers.22.feed_forward.down_proj": {"inputs": [[[14.4375]]], "outputs": [[[12.0]], [[80.5]]], "params": {"weight": [[0.7109375]]}}, "language_model.model.layers.23.self_attn.qkv_proj": {"inputs": [[[67.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.23.self_attn.o_proj": {"inputs": [[[6.78125]]], "outputs": [[[50.5]], [[61.5]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.23.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.81640625]], [[14.0]]]}, "language_model.model.layers.23.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[14.0625]]]}, "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.81640625]]]}, "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.6875]]]}, "language_model.model.layers.23.self_attn.attn.impl.k_cache": {"inputs": [[[14.0]]]}, "language_model.model.layers.23.self_attn.attn.impl.v_cache": {"inputs": [[[14.0625]]]}, "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.5625]], [[14.0]], [[14.0625]]], "outputs": [[[6.78125]], [[1.0]]]}, "language_model.model.layers.23.feed_forward.router": {"inputs": [[[23.625]]], "params": {"weight": [[0.474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op": {"inputs": [[[23.625]]], "outputs": [[[5.6875]], [[68.5]], [[8.5]], [[57.5]], [[44.5]], [[6.09375]], [[34.0]], [[25.125]], [[26.5]], [[53.0]], [[27.75]], [[72.5]], [[56.5]], [[31.625]], [[9.625]], [[48.0]], [[42.5]], [[34.25]], [[17.0]], [[26.875]], [[78.5]], [[22.375]], [[43.25]], [[8.25]], [[22.5]], [[48.0]], [[12.3125]], [[42.0]], [[62.25]], [[7.96875]], [[14.0]], [[11.9375]], [[25.875]], [[16.75]], [[16.875]], [[23.75]], [[51.5]], [[38.0]], [[27.25]], [[13.9375]], [[67.5]], [[40.0]], [[11.625]], [[52.0]], [[23.5]], [[10.0625]], [[7.6875]], [[107.0]], [[37.0]], [[11.5625]], [[76.0]], [[11.0625]], [[57.0]], [[67.0]], [[20.875]], [[20.375]], [[29.25]], [[27.25]], [[8.125]], [[51.5]], [[38.5]], [[62.0]], [[40.75]], [[8.6875]], [[20.25]], [[18.25]], [[23.0]], [[40.75]], [[71.0]], [[32.0]], [[59.0]], [[10.5]], [[22.875]], [[26.375]], [[60.75]], [[13.3125]], [[46.75]], [[59.5]], [[56.5]], [[46.5]], [[33.5]], [[21.0]], [[27.5]], [[40.25]], [[38.0]], [[7.03125]], [[38.0]], [[15.0]], [[7.65625]], [[28.5]], [[18.5]], [[33.25]], [[72.0]], [[28.375]], [[23.25]], [[14.875]], [[63.75]], [[26.375]], [[31.0]], [[18.25]], [[43.75]], [[11.875]], [[15.5625]], [[11.8125]], [[14.5]], [[47.5]], [[48.75]], [[79.5]], [[60.5]], [[8.0]], [[78.0]], [[30.625]], [[52.0]], [[12.3125]], [[59.75]], [[27.25]], [[15.6875]], [[14.3125]], [[90.0]], [[19.75]], [[14.9375]], [[18.75]], [[70.5]], [[37.75]], [[53.25]], [[22.0]], [[24.25]], [[11.6875]], [[8.75]]]}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.431640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.36328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[23.625]]], "params": {"weight": [[0.515625]]}}, "language_model.model.layers.23.feed_forward.shared_expert.down_proj": {"inputs": [[[23.625]]], "outputs": [[[18.625]], [[1.869226869072538e+35]]], "params": {"weight": [[0.71875]]}}, "language_model.model.layers.24.self_attn.qkv_proj": {"inputs": [[[60.5]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.24.self_attn.o_proj": {"inputs": [[[8.1875]]], "outputs": [[[36.0]], [[42.5]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.24.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.34375]], [[22.5]]]}, "language_model.model.layers.24.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.375]]]}, "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.34375]]]}, "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.59375]]]}, "language_model.model.layers.24.self_attn.attn.impl.k_cache": {"inputs": [[[22.5]]]}, "language_model.model.layers.24.self_attn.attn.impl.v_cache": {"inputs": [[[11.375]]]}, "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[16.125]], [[22.5]], [[11.375]]], "outputs": [[[8.1875]], [[1.0]]]}, "language_model.model.layers.24.feed_forward.gate_up_proj": {"inputs": [[[40.5]]], "params": {"weight": [[0.4296875]]}}, "language_model.model.layers.24.feed_forward.down_proj": {"inputs": [[[28.0]]], "outputs": [[[13.3125]], [[62.5]]], "params": {"weight": [[0.8203125]]}}, "language_model.model.layers.25.self_attn.qkv_proj": {"inputs": [[[49.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.25.self_attn.o_proj": {"inputs": [[[6.21875]]], "outputs": [[[47.75]], [[57.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.25.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.171875]], [[19.25]]]}, "language_model.model.layers.25.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.625]]]}, "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.171875]]]}, "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.46875]]]}, "language_model.model.layers.25.self_attn.attn.impl.k_cache": {"inputs": [[[19.25]]]}, "language_model.model.layers.25.self_attn.attn.impl.v_cache": {"inputs": [[[6.625]]]}, "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.8125]], [[19.25]], [[6.625]]], "outputs": [[[6.21875]], [[1.0]]]}, "language_model.model.layers.25.feed_forward.router": {"inputs": [[[30.375]]], "params": {"weight": [[0.412109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op": {"inputs": [[[30.375]]], "outputs": [[[7.25]], [[52.75]], [[28.25]], [[21.25]], [[10.25]], [[42.25]], [[38.25]], [[18.375]], [[75.5]], [[17.75]], [[24.5]], [[6.34375]], [[29.875]], [[37.0]], [[62.75]], [[48.25]], [[27.25]], [[18.875]], [[21.0]], [[46.25]], [[44.25]], [[26.375]], [[53.0]], [[68.5]], [[48.5]], [[11.25]], [[24.5]], [[50.75]], [[10.75]], [[9.8125]], [[42.25]], [[13.3125]], [[13.8125]], [[24.875]], [[61.75]], [[20.75]], [[9.1875]], [[12.9375]], [[21.875]], [[11.125]], [[56.0]], [[23.625]], [[81.0]], [[37.25]], [[60.75]], [[14.25]], [[49.5]], [[104.0]], [[19.25]], [[75.5]], [[27.0]], [[32.0]], [[12.75]], [[8.625]], [[13.625]], [[33.25]], [[37.0]], [[20.375]], [[16.625]], [[32.25]], [[9.0]], [[19.625]], [[42.0]], [[20.25]], [[16.75]], [[107.0]], [[17.875]], [[47.5]], [[32.5]], [[16.25]], [[14.0]], [[17.25]], [[13.625]], [[31.875]], [[31.875]], [[61.5]], [[72.5]], [[9.9375]], [[10.625]], [[21.125]], [[57.0]], [[16.125]], [[50.0]], [[39.75]], [[39.5]], [[7.25]], [[24.25]], [[42.75]], [[63.0]], [[15.0]], [[55.0]], [[28.625]], [[13.125]], [[46.0]], [[23.75]], [[9.5625]], [[47.0]], [[44.5]], [[45.0]], [[12.0]], [[29.5]], [[14.0625]], [[25.125]], [[40.75]], [[12.5625]], [[15.0625]], [[21.25]], [[38.25]], [[6.25]], [[52.0]], [[26.0]], [[37.0]], [[29.75]], [[44.0]], [[38.5]], [[26.25]], [[40.5]], [[65.0]], [[14.8125]], [[78.0]], [[43.75]], [[39.0]], [[41.75]], [[26.5]], [[29.375]], [[27.875]], [[8.4375]], [[11.6875]], [[10.0]]]}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.201171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.46875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.26953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.62109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[30.375]]], "params": {"weight": [[0.4140625]]}}, "language_model.model.layers.25.feed_forward.shared_expert.down_proj": {"inputs": [[[233.0]]], "outputs": [[[135.0]], [[1.869226869072538e+35]]], "params": {"weight": [[0.73046875]]}}, "language_model.model.layers.26.self_attn.qkv_proj": {"inputs": [[[63.25]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.26.self_attn.o_proj": {"inputs": [[[6.9375]]], "outputs": [[[29.0]], [[53.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.26.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.03125]], [[17.25]]]}, "language_model.model.layers.26.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.125]]]}, "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.03125]]]}, "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.0625]]]}, "language_model.model.layers.26.self_attn.attn.impl.k_cache": {"inputs": [[[17.25]]]}, "language_model.model.layers.26.self_attn.attn.impl.v_cache": {"inputs": [[[9.125]]]}, "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.8125]], [[17.25]], [[9.125]]], "outputs": [[[6.9375]], [[1.0]]]}, "language_model.model.layers.26.feed_forward.gate_up_proj": {"inputs": [[[45.5]]], "params": {"weight": [[0.44140625]]}}, "language_model.model.layers.26.feed_forward.down_proj": {"inputs": [[[44.25]]], "outputs": [[[15.25]], [[103.0]]], "params": {"weight": [[0.765625]]}}, "language_model.model.layers.27.self_attn.qkv_proj": {"inputs": [[[72.5]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.27.self_attn.o_proj": {"inputs": [[[5.6875]]], "outputs": [[[60.0]], [[66.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.27.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.203125]], [[16.75]]]}, "language_model.model.layers.27.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.5]]]}, "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.203125]]]}, "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.53125]]]}, "language_model.model.layers.27.self_attn.attn.impl.k_cache": {"inputs": [[[16.75]]]}, "language_model.model.layers.27.self_attn.attn.impl.v_cache": {"inputs": [[[12.5]]]}, "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.8125]], [[16.75]], [[12.5]]], "outputs": [[[5.6875]], [[1.0]]]}, "language_model.model.layers.27.feed_forward.router": {"inputs": [[[30.875]]], "params": {"weight": [[0.353515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op": {"inputs": [[[30.875]]], "outputs": [[[6.09375]], [[15.5]], [[28.375]], [[28.125]], [[25.75]], [[12.0]], [[44.25]], [[35.75]], [[28.5]], [[45.25]], [[31.25]], [[30.5]], [[50.0]], [[42.25]], [[21.375]], [[54.0]], [[39.0]], [[17.75]], [[59.0]], [[36.5]], [[18.875]], [[26.75]], [[15.3125]], [[18.75]], [[14.3125]], [[25.625]], [[18.125]], [[54.25]], [[29.5]], [[14.3125]], [[31.5]], [[81.0]], [[13.5625]], [[51.0]], [[11.8125]], [[87.5]], [[13.0625]], [[27.125]], [[54.75]], [[40.0]], [[37.5]], [[27.125]], [[38.25]], [[44.0]], [[32.0]], [[12.625]], [[40.75]], [[21.375]], [[36.5]], [[30.375]], [[13.0625]], [[34.75]], [[8.375]], [[53.5]], [[39.0]], [[36.5]], [[37.0]], [[54.0]], [[19.875]], [[34.25]], [[11.625]], [[17.875]], [[44.75]], [[45.0]], [[10.6875]], [[44.25]], [[90.0]], [[22.375]], [[17.5]], [[23.5]], [[11.5625]], [[27.5]], [[18.0]], [[81.5]], [[41.0]], [[23.125]], [[59.25]], [[33.0]], [[30.75]], [[54.5]], [[57.5]], [[45.75]], [[61.75]], [[30.125]], [[39.0]], [[25.875]], [[12.6875]], [[14.0]], [[32.75]], [[17.25]], [[30.5]], [[13.0]], [[78.5]], [[18.75]], [[41.5]], [[28.5]], [[17.625]], [[22.5]], [[48.75]], [[22.25]], [[20.625]], [[38.0]], [[40.0]], [[16.75]], [[29.25]], [[11.625]], [[21.5]], [[12.625]], [[54.0]], [[11.9375]], [[37.5]], [[17.0]], [[47.75]], [[11.125]], [[14.75]], [[36.0]], [[17.875]], [[31.625]], [[43.25]], [[21.375]], [[61.75]], [[14.0]], [[31.125]], [[23.0]], [[103.5]], [[19.75]], [[21.375]], [[14.5625]], [[20.0]]]}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[30.875]]], "params": {"weight": [[0.4453125]]}}, "language_model.model.layers.27.feed_forward.shared_expert.down_proj": {"inputs": [[[27.375]]], "outputs": [[[8.875]], [[1.8073924573566552e+29]]], "params": {"weight": [[0.55859375]]}}, "language_model.model.layers.28.self_attn.qkv_proj": {"inputs": [[[77.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.28.self_attn.o_proj": {"inputs": [[[5.90625]]], "outputs": [[[72.0]], [[74.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.28.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.1796875]], [[16.125]]]}, "language_model.model.layers.28.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[14.4375]]]}, "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.1796875]]]}, "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.34375]]]}, "language_model.model.layers.28.self_attn.attn.impl.k_cache": {"inputs": [[[16.125]]]}, "language_model.model.layers.28.self_attn.attn.impl.v_cache": {"inputs": [[[14.4375]]]}, "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.9375]], [[16.125]], [[14.4375]]], "outputs": [[[5.90625]], [[1.0]]]}, "language_model.model.layers.28.feed_forward.gate_up_proj": {"inputs": [[[40.25]]], "params": {"weight": [[0.41015625]]}}, "language_model.model.layers.28.feed_forward.down_proj": {"inputs": [[[64.5]]], "outputs": [[[95.0]], [[103.5]]], "params": {"weight": [[0.8203125]]}}, "language_model.model.layers.29.self_attn.qkv_proj": {"inputs": [[[66.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.29.self_attn.o_proj": {"inputs": [[[5.09375]]], "outputs": [[[70.5]], [[76.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.29.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.078125]], [[16.125]]]}, "language_model.model.layers.29.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.6875]]]}, "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.078125]]]}, "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[3.859375]]]}, "language_model.model.layers.29.self_attn.attn.impl.k_cache": {"inputs": [[[16.125]]]}, "language_model.model.layers.29.self_attn.attn.impl.v_cache": {"inputs": [[[8.6875]]]}, "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.375]], [[16.125]], [[8.6875]]], "outputs": [[[5.09375]], [[1.0]]]}, "language_model.model.layers.29.feed_forward.router": {"inputs": [[[31.75]]], "params": {"weight": [[0.333984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op": {"inputs": [[[31.75]]], "outputs": [[[7.625]], [[23.375]], [[37.75]], [[46.5]], [[38.5]], [[18.375]], [[32.5]], [[16.125]], [[60.75]], [[33.75]], [[64.5]], [[10.3125]], [[14.375]], [[21.375]], [[46.25]], [[40.5]], [[9.0625]], [[88.5]], [[28.75]], [[15.875]], [[19.125]], [[23.0]], [[12.25]], [[36.5]], [[17.875]], [[78.5]], [[25.125]], [[32.5]], [[23.125]], [[25.125]], [[52.5]], [[47.5]], [[13.3125]], [[89.5]], [[51.25]], [[18.25]], [[27.75]], [[30.5]], [[48.0]], [[79.0]], [[43.0]], [[25.125]], [[33.25]], [[12.25]], [[53.25]], [[20.0]], [[27.375]], [[22.125]], [[33.0]], [[35.75]], [[18.875]], [[20.5]], [[42.75]], [[41.0]], [[46.25]], [[27.875]], [[28.75]], [[69.5]], [[32.5]], [[25.75]], [[16.625]], [[44.5]], [[12.0625]], [[20.625]], [[56.5]], [[86.5]], [[54.25]], [[44.25]], [[54.75]], [[46.75]], [[23.625]], [[22.125]], [[39.5]], [[26.0]], [[27.625]], [[13.875]], [[46.25]], [[32.75]], [[45.75]], [[47.25]], [[28.75]], [[28.5]], [[14.4375]], [[22.75]], [[32.0]], [[15.5]], [[22.125]], [[18.0]], [[17.375]], [[19.0]], [[11.25]], [[31.0]], [[33.5]], [[51.75]], [[52.25]], [[14.6875]], [[18.875]], [[37.25]], [[27.75]], [[37.0]], [[15.5]], [[33.0]], [[30.125]], [[32.25]], [[60.75]], [[16.375]], [[123.5]], [[61.5]], [[25.25]], [[28.5]], [[40.5]], [[24.875]], [[46.75]], [[54.5]], [[14.0625]], [[25.75]], [[43.5]], [[32.75]], [[38.5]], [[94.5]], [[22.25]], [[56.75]], [[22.625]], [[56.0]], [[36.25]], [[17.0]], [[21.5]], [[91.5]], [[21.875]]]}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.07958984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.08056640625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[31.75]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.29.feed_forward.shared_expert.down_proj": {"inputs": [[[34.5]]], "outputs": [[[22.0]], [[1.188422437713965e+29]]], "params": {"weight": [[0.74609375]]}}, "language_model.model.layers.30.self_attn.qkv_proj": {"inputs": [[[71.5]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.30.self_attn.o_proj": {"inputs": [[[9.4375]]], "outputs": [[[52.25]], [[57.0]]], "params": {"weight": [[0.80859375]]}}, "language_model.model.layers.30.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.125]], [[15.3125]]]}, "language_model.model.layers.30.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.5]]]}, "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.125]]]}, "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[8.5625]]]}, "language_model.model.layers.30.self_attn.attn.impl.k_cache": {"inputs": [[[15.3125]]]}, "language_model.model.layers.30.self_attn.attn.impl.v_cache": {"inputs": [[[11.5]]]}, "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.125]], [[15.3125]], [[11.5]]], "outputs": [[[9.4375]], [[1.0]]]}, "language_model.model.layers.30.feed_forward.gate_up_proj": {"inputs": [[[32.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.30.feed_forward.down_proj": {"inputs": [[[31.25]]], "outputs": [[[18.625]], [[139.0]]], "params": {"weight": [[0.70703125]]}}, "language_model.model.layers.31.self_attn.qkv_proj": {"inputs": [[[71.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.31.self_attn.o_proj": {"inputs": [[[9.625]]], "outputs": [[[66.5]], [[88.0]]], "params": {"weight": [[0.46484375]]}}, "language_model.model.layers.31.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.81640625]], [[8.3125]]]}, "language_model.model.layers.31.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.875]]]}, "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.81640625]]]}, "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[9.625]]]}, "language_model.model.layers.31.self_attn.attn.impl.k_cache": {"inputs": [[[8.3125]]]}, "language_model.model.layers.31.self_attn.attn.impl.v_cache": {"inputs": [[[9.875]]]}, "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.5]], [[8.3125]], [[9.875]]], "outputs": [[[9.0625]], [[1.0]]]}, "language_model.model.layers.31.feed_forward.router": {"inputs": [[[33.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op": {"inputs": [[[33.0]]], "outputs": [[[9.25]], [[17.375]], [[142.0]], [[20.0]], [[47.0]], [[45.0]], [[33.25]], [[63.0]], [[12.375]], [[24.125]], [[37.0]], [[38.5]], [[13.125]], [[24.375]], [[12.4375]], [[17.625]], [[13.0]], [[21.625]], [[32.5]], [[62.0]], [[19.25]], [[10.5]], [[24.875]], [[30.0]], [[72.5]], [[62.5]], [[47.25]], [[28.5]], [[40.25]], [[13.5625]], [[13.5625]], [[20.875]], [[24.75]], [[28.375]], [[101.0]], [[11.25]], [[19.125]], [[29.875]], [[26.875]], [[30.875]], [[64.5]], [[68.5]], [[62.75]], [[16.75]], [[59.5]], [[81.5]], [[50.0]], [[70.5]], [[21.125]], [[66.0]], [[45.0]], [[41.5]], [[36.0]], [[50.75]], [[23.25]], [[38.5]], [[22.5]], [[14.25]], [[91.5]], [[25.25]], [[38.25]], [[27.5]], [[39.75]], [[27.5]], [[36.25]], [[48.0]], [[33.0]], [[20.75]], [[39.5]], [[36.0]], [[15.0]], [[36.25]], [[52.5]], [[19.125]], [[42.75]], [[13.0]], [[12.9375]], [[35.75]], [[45.25]], [[35.25]], [[57.25]], [[31.875]], [[74.0]], [[18.5]], [[82.5]], [[28.625]], [[46.25]], [[45.75]], [[29.75]], [[70.0]], [[80.5]], [[61.5]], [[34.5]], [[46.75]], [[12.8125]], [[36.5]], [[39.75]], [[30.5]], [[15.875]], [[16.375]], [[28.5]], [[17.375]], [[14.1875]], [[54.25]], [[26.875]], [[16.125]], [[6.46875]], [[27.625]], [[11.875]], [[31.75]], [[62.0]], [[38.25]], [[43.25]], [[39.5]], [[26.75]], [[26.25]], [[45.25]], [[69.0]], [[22.125]], [[20.5]], [[88.0]], [[27.25]], [[22.75]], [[10.0625]], [[12.75]], [[25.5]], [[19.25]], [[20.125]], [[28.75]]]}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.2333984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.240234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.447265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.34765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.33203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.2265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.43359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.345703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[33.0]]], "params": {"weight": [[0.55078125]]}}, "language_model.model.layers.31.feed_forward.shared_expert.down_proj": {"inputs": [[[47.5]]], "outputs": [[[13.75]], [[1.895188353365212e+35]]], "params": {"weight": [[0.70703125]]}}, "language_model.model.layers.32.self_attn.qkv_proj": {"inputs": [[[53.25]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.32.self_attn.o_proj": {"inputs": [[[7.84375]]], "outputs": [[[96.0]], [[105.0]]], "params": {"weight": [[0.5546875]]}}, "language_model.model.layers.32.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.90234375]], [[23.625]]]}, "language_model.model.layers.32.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.375]]]}, "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.90234375]]]}, "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.96875]]]}, "language_model.model.layers.32.self_attn.attn.impl.k_cache": {"inputs": [[[23.625]]]}, "language_model.model.layers.32.self_attn.attn.impl.v_cache": {"inputs": [[[9.375]]]}, "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.125]], [[23.625]], [[9.25]]], "outputs": [[[7.84375]], [[1.0]]]}, "language_model.model.layers.32.feed_forward.gate_up_proj": {"inputs": [[[32.75]]], "params": {"weight": [[0.486328125]]}}, "language_model.model.layers.32.feed_forward.down_proj": {"inputs": [[[43.0]]], "outputs": [[[21.125]], [[79.5]]], "params": {"weight": [[0.65234375]]}}, "language_model.model.layers.33.self_attn.qkv_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.33.self_attn.o_proj": {"inputs": [[[4.90625]]], "outputs": [[[97.0]], [[118.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.33.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.3046875]], [[13.375]]]}, "language_model.model.layers.33.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[7.15625]]]}, "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.3046875]]]}, "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[5.21875]]]}, "language_model.model.layers.33.self_attn.attn.impl.k_cache": {"inputs": [[[13.9375]]]}, "language_model.model.layers.33.self_attn.attn.impl.v_cache": {"inputs": [[[7.15625]]]}, "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[17.125]], [[13.9375]], [[7.15625]]], "outputs": [[[4.90625]], [[1.0]]]}, "language_model.model.layers.33.feed_forward.router": {"inputs": [[[64.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op": {"inputs": [[[64.0]]], "outputs": [[[12.5]], [[65.0]], [[24.5]], [[30.5]], [[39.75]], [[22.25]], [[76.0]], [[15.3125]], [[71.5]], [[45.75]], [[49.75]], [[17.375]], [[54.25]], [[80.0]], [[24.125]], [[72.5]], [[21.375]], [[33.0]], [[28.0]], [[16.375]], [[15.125]], [[16.875]], [[26.125]], [[19.875]], [[50.0]], [[29.125]], [[33.25]], [[94.0]], [[30.375]], [[50.5]], [[38.25]], [[41.75]], [[35.0]], [[17.875]], [[101.0]], [[37.25]], [[31.0]], [[24.0]], [[16.75]], [[72.5]], [[24.25]], [[16.875]], [[25.5]], [[61.75]], [[112.5]], [[38.25]], [[19.75]], [[52.0]], [[41.0]], [[49.25]], [[59.0]], [[63.0]], [[14.3125]], [[32.25]], [[15.9375]], [[182.0]], [[86.5]], [[59.75]], [[18.375]], [[24.375]], [[82.0]], [[41.5]], [[55.5]], [[44.0]], [[78.0]], [[31.0]], [[76.0]], [[20.875]], [[19.5]], [[45.25]], [[56.0]], [[76.5]], [[53.5]], [[92.5]], [[76.0]], [[27.125]], [[43.5]], [[47.5]], [[31.25]], [[14.5]], [[31.75]], [[22.25]], [[8.6875]], [[53.0]], [[27.375]], [[24.125]], [[114.0]], [[60.75]], [[22.75]], [[79.0]], [[47.5]], [[14.625]], [[18.5]], [[21.5]], [[34.25]], [[83.5]], [[72.0]], [[20.5]], [[73.0]], [[66.5]], [[41.0]], [[72.5]], [[63.5]], [[42.25]], [[39.75]], [[76.0]], [[38.5]], [[47.25]], [[26.0]], [[21.375]], [[30.875]], [[57.0]], [[124.5]], [[17.5]], [[34.25]], [[47.5]], [[18.125]], [[16.375]], [[43.25]], [[33.75]], [[81.0]], [[32.75]], [[34.25]], [[22.375]], [[11.4375]], [[46.0]], [[33.25]], [[15.75]], [[55.0]]]}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.263671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.07861328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.224609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.080078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[64.0]]], "params": {"weight": [[0.373046875]]}}, "language_model.model.layers.33.feed_forward.shared_expert.down_proj": {"inputs": [[[53.25]]], "outputs": [[[25.75]], [[1.6096120261457966e+35]]], "params": {"weight": [[0.66015625]]}}, "language_model.model.layers.34.self_attn.qkv_proj": {"inputs": [[[54.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.34.self_attn.o_proj": {"inputs": [[[7.09375]]], "outputs": [[[46.75]], [[80.5]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.34.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.2734375]], [[12.6875]]]}, "language_model.model.layers.34.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[8.1875]]]}, "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.2734375]]]}, "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.53125]]]}, "language_model.model.layers.34.self_attn.attn.impl.k_cache": {"inputs": [[[12.6875]]]}, "language_model.model.layers.34.self_attn.attn.impl.v_cache": {"inputs": [[[8.1875]]]}, "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[15.1875]], [[12.6875]], [[8.1875]]], "outputs": [[[7.09375]], [[1.0]]]}, "language_model.model.layers.34.feed_forward.gate_up_proj": {"inputs": [[[31.125]]], "params": {"weight": [[0.291015625]]}}, "language_model.model.layers.34.feed_forward.down_proj": {"inputs": [[[62.75]]], "outputs": [[[30.375]], [[128.0]]], "params": {"weight": [[0.41796875]]}}, "language_model.model.layers.35.self_attn.qkv_proj": {"inputs": [[[52.75]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.35.self_attn.o_proj": {"inputs": [[[6.1875]]], "outputs": [[[87.5]], [[93.0]]], "params": {"weight": [[0.216796875]]}}, "language_model.model.layers.35.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.96484375]], [[13.75]]]}, "language_model.model.layers.35.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[10.5]]]}, "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.96484375]]]}, "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.5]]]}, "language_model.model.layers.35.self_attn.attn.impl.k_cache": {"inputs": [[[13.75]]]}, "language_model.model.layers.35.self_attn.attn.impl.v_cache": {"inputs": [[[10.5]]]}, "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[10.75]], [[13.75]], [[10.5]]], "outputs": [[[5.40625]], [[1.0]]]}, "language_model.model.layers.35.feed_forward.router": {"inputs": [[[64.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op": {"inputs": [[[64.0]]], "outputs": [[[10.625]], [[145.0]], [[18.875]], [[26.0]], [[28.25]], [[105.0]], [[17.75]], [[80.5]], [[143.0]], [[67.5]], [[62.5]], [[76.5]], [[26.75]], [[126.5]], [[34.75]], [[76.5]], [[53.5]], [[67.5]], [[44.25]], [[153.0]], [[61.75]], [[38.25]], [[24.625]], [[22.5]], [[27.5]], [[68.0]], [[47.0]], [[93.5]], [[58.5]], [[31.75]], [[51.5]], [[137.0]], [[51.75]], [[52.0]], [[35.75]], [[67.5]], [[139.0]], [[47.25]], [[51.5]], [[57.0]], [[53.75]], [[32.75]], [[41.0]], [[26.5]], [[43.75]], [[112.0]], [[23.375]], [[20.0]], [[31.75]], [[64.0]], [[79.0]], [[33.25]], [[52.0]], [[35.75]], [[29.625]], [[20.375]], [[30.125]], [[86.5]], [[38.75]], [[85.0]], [[74.5]], [[68.5]], [[21.375]], [[59.5]], [[32.5]], [[37.25]], [[34.75]], [[102.5]], [[37.75]], [[22.625]], [[94.0]], [[68.0]], [[151.0]], [[34.5]], [[39.75]], [[36.5]], [[33.25]], [[45.0]], [[60.75]], [[155.0]], [[130.0]], [[70.0]], [[99.0]], [[34.5]], [[40.0]], [[36.25]], [[142.0]], [[42.25]], [[27.5]], [[24.125]], [[29.75]], [[107.0]], [[62.5]], [[90.5]], [[114.5]], [[92.5]], [[60.5]], [[65.5]], [[44.5]], [[117.5]], [[60.25]], [[107.5]], [[18.875]], [[26.625]], [[32.25]], [[20.875]], [[63.75]], [[42.0]], [[47.25]], [[103.0]], [[62.25]], [[52.25]], [[43.5]], [[50.75]], [[38.25]], [[93.0]], [[68.5]], [[34.75]], [[21.125]], [[59.25]], [[55.0]], [[71.0]], [[20.875]], [[33.25]], [[86.5]], [[72.5]], [[30.75]], [[28.25]], [[45.25]]]}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.12451171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.361328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.396484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.3359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[64.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.35.feed_forward.shared_expert.down_proj": {"inputs": [[[87.5]]], "outputs": [[[12.1875]], [[1.895188353365212e+35]]], "params": {"weight": [[0.5546875]]}}, "language_model.model.layers.36.self_attn.qkv_proj": {"inputs": [[[56.75]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.36.self_attn.o_proj": {"inputs": [[[5.8125]]], "outputs": [[[27.75]], [[43.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.36.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.25]], [[13.6875]]]}, "language_model.model.layers.36.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[6.8125]]]}, "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.25]]]}, "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[4.90625]]]}, "language_model.model.layers.36.self_attn.attn.impl.k_cache": {"inputs": [[[13.6875]]]}, "language_model.model.layers.36.self_attn.attn.impl.v_cache": {"inputs": [[[6.8125]]]}, "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[18.375]], [[13.6875]], [[6.8125]]], "outputs": [[[5.8125]], [[1.0]]]}, "language_model.model.layers.36.feed_forward.gate_up_proj": {"inputs": [[[29.625]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.36.feed_forward.down_proj": {"inputs": [[[72.5]]], "outputs": [[[38.25]], [[76.5]]], "params": {"weight": [[0.56640625]]}}, "language_model.model.layers.37.self_attn.qkv_proj": {"inputs": [[[50.25]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.37.self_attn.o_proj": {"inputs": [[[8.4375]]], "outputs": [[[102.0]], [[105.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.37.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.6484375]], [[13.0625]]]}, "language_model.model.layers.37.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[11.625]]]}, "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.6484375]]]}, "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[8.4375]]]}, "language_model.model.layers.37.self_attn.attn.impl.k_cache": {"inputs": [[[13.4375]]]}, "language_model.model.layers.37.self_attn.attn.impl.v_cache": {"inputs": [[[11.625]]]}, "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[21.0]], [[13.4375]], [[11.625]]], "outputs": [[[8.25]], [[1.0]]]}, "language_model.model.layers.37.feed_forward.router": {"inputs": [[[70.0]]], "params": {"weight": [[0.279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op": {"inputs": [[[70.0]]], "outputs": [[[13.1875]], [[209.0]], [[80.0]], [[74.0]], [[93.0]], [[43.0]], [[56.5]], [[67.5]], [[72.5]], [[21.75]], [[119.5]], [[30.875]], [[24.5]], [[41.75]], [[89.0]], [[67.5]], [[41.25]], [[62.5]], [[60.25]], [[85.0]], [[83.5]], [[83.5]], [[60.5]], [[79.0]], [[43.75]], [[105.5]], [[91.5]], [[46.75]], [[70.5]], [[131.0]], [[65.0]], [[20.5]], [[104.0]], [[32.75]], [[23.0]], [[69.0]], [[64.0]], [[64.5]], [[114.5]], [[141.0]], [[53.5]], [[63.0]], [[29.0]], [[24.0]], [[50.75]], [[37.0]], [[91.5]], [[105.5]], [[23.125]], [[65.0]], [[24.0]], [[38.25]], [[61.0]], [[62.75]], [[60.25]], [[30.0]], [[112.0]], [[88.0]], [[46.25]], [[98.5]], [[114.5]], [[116.0]], [[47.0]], [[55.75]], [[53.25]], [[74.5]], [[37.0]], [[42.75]], [[111.0]], [[39.75]], [[37.5]], [[60.0]], [[36.25]], [[52.75]], [[44.0]], [[27.75]], [[75.5]], [[124.5]], [[35.25]], [[50.75]], [[93.0]], [[66.0]], [[50.25]], [[105.5]], [[35.0]], [[34.5]], [[34.0]], [[96.0]], [[101.5]], [[139.0]], [[118.5]], [[48.0]], [[56.5]], [[28.875]], [[36.5]], [[34.25]], [[70.5]], [[26.375]], [[32.25]], [[93.0]], [[125.5]], [[139.0]], [[96.0]], [[94.5]], [[28.375]], [[103.0]], [[55.0]], [[89.5]], [[21.625]], [[74.0]], [[64.5]], [[45.5]], [[61.5]], [[106.0]], [[86.5]], [[49.0]], [[86.0]], [[67.0]], [[94.5]], [[36.25]], [[67.5]], [[25.125]], [[38.25]], [[55.25]], [[57.0]], [[149.0]], [[24.375]], [[46.0]], [[66.0]]]}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.73046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.318359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.2109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.2451171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.2275390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.2255859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.2578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.21484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.23046875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.35546875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[70.0]]], "params": {"weight": [[0.46484375]]}}, "language_model.model.layers.37.feed_forward.shared_expert.down_proj": {"inputs": [[[104.0]]], "outputs": [[[44.25]], [[1.895188353365212e+35]]], "params": {"weight": [[1.046875]]}}, "language_model.model.layers.38.self_attn.qkv_proj": {"inputs": [[[47.25]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.38.self_attn.o_proj": {"inputs": [[[8.5625]]], "outputs": [[[26.75]], [[56.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.38.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.859375]], [[11.25]]]}, "language_model.model.layers.38.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.1875]]]}, "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.859375]]]}, "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[8.5625]]]}, "language_model.model.layers.38.self_attn.attn.impl.k_cache": {"inputs": [[[11.375]]]}, "language_model.model.layers.38.self_attn.attn.impl.v_cache": {"inputs": [[[12.1875]]]}, "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[22.0]], [[11.375]], [[12.1875]]], "outputs": [[[7.21875]], [[1.0]]]}, "language_model.model.layers.38.feed_forward.gate_up_proj": {"inputs": [[[31.125]]], "params": {"weight": [[0.515625]]}}, "language_model.model.layers.38.feed_forward.down_proj": {"inputs": [[[110.0]]], "outputs": [[[24.75]], [[193.0]]], "params": {"weight": [[0.51171875]]}}, "language_model.model.layers.39.self_attn.qkv_proj": {"inputs": [[[49.75]]], "params": {"weight": [[0.4296875]]}}, "language_model.model.layers.39.self_attn.o_proj": {"inputs": [[[9.9375]]], "outputs": [[[129.0]], [[134.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.39.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.5]], [[11.9375]]]}, "language_model.model.layers.39.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[21.5]]]}, "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.5]]]}, "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[13.4375]]]}, "language_model.model.layers.39.self_attn.attn.impl.k_cache": {"inputs": [[[11.9375]]]}, "language_model.model.layers.39.self_attn.attn.impl.v_cache": {"inputs": [[[21.5]]]}, "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[18.25]], [[11.9375]], [[21.5]]], "outputs": [[[9.9375]], [[1.0]]]}, "language_model.model.layers.39.feed_forward.router": {"inputs": [[[61.5]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op": {"inputs": [[[61.5]]], "outputs": [[[13.8125]], [[54.0]], [[61.0]], [[72.5]], [[92.0]], [[54.75]], [[79.0]], [[54.75]], [[84.5]], [[32.75]], [[54.5]], [[57.0]], [[29.625]], [[142.0]], [[122.0]], [[57.75]], [[55.25]], [[75.0]], [[92.5]], [[53.25]], [[89.0]], [[45.5]], [[108.5]], [[44.75]], [[84.0]], [[63.0]], [[211.0]], [[102.0]], [[103.0]], [[115.5]], [[52.75]], [[82.5]], [[80.5]], [[26.5]], [[50.5]], [[173.0]], [[81.0]], [[172.0]], [[65.5]], [[138.0]], [[53.5]], [[53.5]], [[48.25]], [[68.0]], [[43.5]], [[50.0]], [[47.0]], [[58.5]], [[33.5]], [[84.0]], [[89.5]], [[93.5]], [[71.5]], [[52.75]], [[62.25]], [[62.0]], [[87.5]], [[33.75]], [[73.0]], [[50.5]], [[51.5]], [[91.0]], [[99.5]], [[81.0]], [[18.125]], [[35.75]], [[67.0]], [[54.5]], [[82.5]], [[85.5]], [[92.0]], [[93.0]], [[36.75]], [[62.75]], [[218.0]], [[63.25]], [[67.5]], [[84.5]], [[68.5]], [[85.0]], [[53.5]], [[52.25]], [[72.5]], [[98.5]], [[43.0]], [[89.5]], [[186.0]], [[141.0]], [[46.75]], [[104.0]], [[82.0]], [[37.25]], [[102.0]], [[49.0]], [[66.0]], [[58.25]], [[38.5]], [[117.0]], [[83.5]], [[38.25]], [[77.0]], [[38.5]], [[65.5]], [[94.5]], [[185.0]], [[88.5]], [[89.5]], [[94.0]], [[113.5]], [[78.5]], [[72.0]], [[51.0]], [[42.75]], [[39.5]], [[103.0]], [[46.5]], [[112.5]], [[30.0]], [[33.5]], [[94.5]], [[76.0]], [[52.5]], [[76.0]], [[83.5]], [[27.5]], [[69.0]], [[45.5]], [[99.0]], [[64.5]]]}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.208984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1982421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.30859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.2060546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[61.5]]], "params": {"weight": [[0.3984375]]}}, "language_model.model.layers.39.feed_forward.shared_expert.down_proj": {"inputs": [[[127.5]]], "outputs": [[[16.75]], [[1.895188353365212e+35]]], "params": {"weight": [[0.40625]]}}, "language_model.model.layers.40.self_attn.qkv_proj": {"inputs": [[[46.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.40.self_attn.o_proj": {"inputs": [[[11.4375]]], "outputs": [[[93.0]], [[90.0]]], "params": {"weight": [[0.455078125]]}}, "language_model.model.layers.40.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.140625]], [[21.625]]]}, "language_model.model.layers.40.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[12.375]]]}, "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.140625]]]}, "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[10.0]]]}, "language_model.model.layers.40.self_attn.attn.impl.k_cache": {"inputs": [[[21.625]]]}, "language_model.model.layers.40.self_attn.attn.impl.v_cache": {"inputs": [[[12.375]]]}, "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[13.3125]], [[21.625]], [[12.375]]], "outputs": [[[11.4375]], [[1.0]]]}, "language_model.model.layers.40.feed_forward.gate_up_proj": {"inputs": [[[40.75]]], "params": {"weight": [[0.4453125]]}}, "language_model.model.layers.40.feed_forward.down_proj": {"inputs": [[[162.0]]], "outputs": [[[22.625]], [[146.0]]], "params": {"weight": [[0.609375]]}}, "language_model.model.layers.41.self_attn.qkv_proj": {"inputs": [[[33.75]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.41.self_attn.o_proj": {"inputs": [[[6.625]]], "outputs": [[[159.0]], [[164.0]]], "params": {"weight": [[0.173828125]]}}, "language_model.model.layers.41.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.078125]], [[18.125]]]}, "language_model.model.layers.41.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[10.875]]]}, "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.078125]]]}, "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[7.25]]]}, "language_model.model.layers.41.self_attn.attn.impl.k_cache": {"inputs": [[[18.125]]]}, "language_model.model.layers.41.self_attn.attn.impl.v_cache": {"inputs": [[[10.875]]]}, "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[12.1875]], [[18.125]], [[10.875]]], "outputs": [[[6.625]], [[1.0]]]}, "language_model.model.layers.41.feed_forward.router": {"inputs": [[[68.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op": {"inputs": [[[68.0]]], "outputs": [[[18.0]], [[56.25]], [[65.0]], [[42.5]], [[82.0]], [[51.5]], [[45.5]], [[181.0]], [[46.75]], [[53.75]], [[78.5]], [[91.5]], [[45.5]], [[79.0]], [[74.5]], [[156.0]], [[95.5]], [[135.0]], [[133.0]], [[109.5]], [[61.5]], [[80.5]], [[45.25]], [[123.5]], [[83.0]], [[177.0]], [[106.5]], [[139.0]], [[53.0]], [[57.25]], [[67.0]], [[61.75]], [[122.5]], [[175.0]], [[52.25]], [[117.5]], [[87.0]], [[47.0]], [[128.0]], [[51.5]], [[56.0]], [[65.0]], [[133.0]], [[46.25]], [[41.25]], [[55.25]], [[136.0]], [[65.0]], [[68.5]], [[148.0]], [[112.0]], [[117.0]], [[68.5]], [[92.5]], [[151.0]], [[139.0]], [[70.5]], [[176.0]], [[68.5]], [[81.5]], [[69.5]], [[117.0]], [[74.0]], [[74.5]], [[70.0]], [[145.0]], [[50.0]], [[110.0]], [[52.5]], [[42.25]], [[155.0]], [[42.0]], [[36.75]], [[70.5]], [[34.25]], [[109.5]], [[136.0]], [[82.0]], [[44.5]], [[74.5]], [[122.5]], [[202.0]], [[120.5]], [[89.0]], [[61.25]], [[75.5]], [[41.5]], [[81.0]], [[38.25]], [[73.5]], [[106.5]], [[74.0]], [[104.0]], [[178.0]], [[112.5]], [[73.0]], [[132.0]], [[49.25]], [[80.5]], [[55.25]], [[64.5]], [[132.0]], [[63.25]], [[98.0]], [[54.25]], [[85.0]], [[58.0]], [[84.5]], [[92.5]], [[57.5]], [[178.0]], [[165.0]], [[89.5]], [[67.0]], [[64.5]], [[92.0]], [[168.0]], [[57.25]], [[149.0]], [[112.5]], [[125.5]], [[77.5]], [[83.5]], [[102.0]], [[36.5]], [[63.75]], [[106.5]], [[83.0]], [[94.5]]]}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1806640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.232421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1884765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.25]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.267578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.4375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.4765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.400390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.416015625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.248046875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.3203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.3671875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.1923828125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[68.0]]], "params": {"weight": [[0.58203125]]}}, "language_model.model.layers.41.feed_forward.shared_expert.down_proj": {"inputs": [[[206.0]]], "outputs": [[[19.75]], [[1.5382179443409427e+35]]], "params": {"weight": [[0.53125]]}}, "language_model.model.layers.42.self_attn.qkv_proj": {"inputs": [[[38.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.42.self_attn.o_proj": {"inputs": [[[13.5625]]], "outputs": [[[50.5]], [[134.0]]], "params": {"weight": [[0.71484375]]}}, "language_model.model.layers.42.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.2265625]], [[18.0]]]}, "language_model.model.layers.42.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[15.5625]]]}, "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.2265625]]]}, "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[12.0]]]}, "language_model.model.layers.42.self_attn.attn.impl.k_cache": {"inputs": [[[18.0]]]}, "language_model.model.layers.42.self_attn.attn.impl.v_cache": {"inputs": [[[15.5625]]]}, "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.75]], [[18.0]], [[15.5625]]], "outputs": [[[13.5625]], [[1.0]]]}, "language_model.model.layers.42.feed_forward.gate_up_proj": {"inputs": [[[43.75]]], "params": {"weight": [[0.62890625]]}}, "language_model.model.layers.42.feed_forward.down_proj": {"inputs": [[[175.0]]], "outputs": [[[59.5]], [[338.0]]], "params": {"weight": [[0.490234375]]}}, "language_model.model.layers.43.self_attn.qkv_proj": {"inputs": [[[45.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.43.self_attn.o_proj": {"inputs": [[[7.3125]]], "outputs": [[[94.5]], [[232.0]]], "params": {"weight": [[0.94921875]]}}, "language_model.model.layers.43.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.046875]], [[7.8125]]]}, "language_model.model.layers.43.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.6875]]]}, "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.046875]]]}, "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.625]]]}, "language_model.model.layers.43.self_attn.attn.impl.k_cache": {"inputs": [[[7.8125]]]}, "language_model.model.layers.43.self_attn.attn.impl.v_cache": {"inputs": [[[9.6875]]]}, "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[11.625]], [[7.8125]], [[9.6875]]], "outputs": [[[7.3125]], [[1.0]]]}, "language_model.model.layers.43.feed_forward.router": {"inputs": [[[68.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op": {"inputs": [[[68.0]]], "outputs": [[[23.625]], [[51.0]], [[100.5]], [[107.0]], [[83.5]], [[118.5]], [[39.5]], [[46.0]], [[206.0]], [[69.0]], [[134.0]], [[53.75]], [[126.5]], [[88.5]], [[64.5]], [[226.0]], [[212.0]], [[74.0]], [[60.5]], [[165.0]], [[53.5]], [[238.0]], [[119.0]], [[166.0]], [[120.5]], [[97.5]], [[96.5]], [[127.0]], [[71.0]], [[159.0]], [[90.0]], [[53.75]], [[80.0]], [[75.0]], [[99.0]], [[89.0]], [[192.0]], [[94.0]], [[153.0]], [[68.0]], [[143.0]], [[60.5]], [[149.0]], [[204.0]], [[72.5]], [[234.0]], [[134.0]], [[68.0]], [[44.5]], [[91.5]], [[107.0]], [[46.25]], [[288.0]], [[115.5]], [[108.0]], [[110.0]], [[64.0]], [[43.25]], [[62.25]], [[92.5]], [[134.0]], [[41.25]], [[81.0]], [[44.25]], [[150.0]], [[131.0]], [[87.5]], [[123.0]], [[49.25]], [[187.0]], [[47.75]], [[66.5]], [[53.0]], [[55.5]], [[106.5]], [[38.5]], [[83.0]], [[182.0]], [[61.5]], [[103.0]], [[66.0]], [[74.5]], [[98.0]], [[164.0]], [[80.0]], [[81.5]], [[130.0]], [[58.5]], [[63.0]], [[101.5]], [[80.5]], [[200.0]], [[62.5]], [[66.5]], [[62.0]], [[43.25]], [[67.0]], [[36.5]], [[135.0]], [[94.5]], [[131.0]], [[95.0]], [[52.0]], [[52.5]], [[64.5]], [[78.5]], [[104.5]], [[79.0]], [[116.5]], [[98.5]], [[78.0]], [[145.0]], [[124.0]], [[124.0]], [[116.0]], [[34.25]], [[113.5]], [[98.0]], [[77.0]], [[90.0]], [[48.0]], [[111.0]], [[157.0]], [[150.0]], [[144.0]], [[85.5]], [[126.0]], [[140.0]], [[50.25]]]}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.12255859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.244140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.2421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.19140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.2099609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1513671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.15234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.2314453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.2236328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.181640625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.1796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.212890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.2294921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.1201171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.1171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.1953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10693359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.12890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.314453125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.255859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.166015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.115234375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.16015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.08203125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[68.0]]], "params": {"weight": [[0.298828125]]}}, "language_model.model.layers.43.feed_forward.shared_expert.down_proj": {"inputs": [[[262.0]]], "outputs": [[[44.25]], [[1.869226869072538e+35]]], "params": {"weight": [[0.71484375]]}}, "language_model.model.layers.44.self_attn.qkv_proj": {"inputs": [[[28.625]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.44.self_attn.o_proj": {"inputs": [[[15.1875]]], "outputs": [[[68.0]], [[153.0]]], "params": {"weight": [[0.2734375]]}}, "language_model.model.layers.44.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.2734375]], [[25.375]]]}, "language_model.model.layers.44.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[17.5]]]}, "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.2734375]]]}, "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[13.6875]]]}, "language_model.model.layers.44.self_attn.attn.impl.k_cache": {"inputs": [[[25.375]]]}, "language_model.model.layers.44.self_attn.attn.impl.v_cache": {"inputs": [[[17.5]]]}, "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[14.9375]], [[25.375]], [[17.5]]], "outputs": [[[15.1875]], [[1.0]]]}, "language_model.model.layers.44.feed_forward.gate_up_proj": {"inputs": [[[41.5]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.44.feed_forward.down_proj": {"inputs": [[[242.0]]], "outputs": [[[63.25]], [[191.0]]], "params": {"weight": [[0.7109375]]}}, "language_model.model.layers.45.self_attn.qkv_proj": {"inputs": [[[27.375]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.45.self_attn.o_proj": {"inputs": [[[16.75]]], "outputs": [[[169.0]], [[220.0]]], "params": {"weight": [[0.39453125]]}}, "language_model.model.layers.45.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.7421875]], [[17.0]]]}, "language_model.model.layers.45.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[16.875]]]}, "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.7421875]]]}, "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[14.4375]]]}, "language_model.model.layers.45.self_attn.attn.impl.k_cache": {"inputs": [[[17.0]]]}, "language_model.model.layers.45.self_attn.attn.impl.v_cache": {"inputs": [[[16.875]]]}, "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[21.5]], [[17.0]], [[16.875]]], "outputs": [[[16.75]], [[1.0]]]}, "language_model.model.layers.45.feed_forward.router": {"inputs": [[[51.75]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op": {"inputs": [[[51.75]]], "outputs": [[[9.0625]], [[79.0]], [[13.25]], [[44.0]], [[86.5]], [[0.0]], [[42.75]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[42.75]], [[32.75]], [[24.125]], [[28.0]], [[0.0]], [[0.0]], [[0.0]], [[35.75]], [[0.0]], [[0.0]], [[0.0]], [[31.625]], [[0.0]], [[39.0]], [[36.0]], [[31.875]], [[50.0]], [[43.25]], [[0.0]], [[24.375]], [[69.0]], [[26.625]], [[38.0]], [[61.0]], [[54.0]], [[0.0]], [[33.0]], [[19.125]], [[43.75]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[117.5]], [[0.0]], [[75.0]], [[0.0]], [[0.0]], [[0.0]], [[32.75]], [[0.0]], [[71.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[35.5]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[38.75]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[56.25]], [[84.0]], [[94.0]], [[30.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[47.25]], [[0.0]], [[63.75]], [[34.0]], [[35.25]], [[84.5]], [[0.0]], [[0.0]], [[18.5]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[0.0]], [[11.3125]], [[5.827586677109586e-18]], [[86.5]], [[0.0]], [[69.5]], [[0.0]], [[0.0]]]}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.10888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1650390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.12109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.11328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1572265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.24609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.2490234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.2138671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.1689453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1591796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.32421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.11083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.271484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.1015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.2412109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.150390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.10546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1005859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.12060546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.126953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.134765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.34375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1357421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.11572265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.419921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1044921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.12158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.1259765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.09423828125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.08837890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.10205078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.193359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.2041015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.10595703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.0986328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0810546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.1484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.08642578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.08154296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.2177734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.158203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.107421875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.08447265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.08544921875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1396484375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.1083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.1181640625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.16796875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.33984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1943359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.083984375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.177734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1103515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.08935546875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09619140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.1123046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.08349609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.0859375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.109375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.0849609375]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.1904296875]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.0830078125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.08251953125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.1220703125]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[51.75]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.45.feed_forward.shared_expert.down_proj": {"inputs": [[[218.0]]], "outputs": [[[29.125]], [[1.6096120261457966e+35]]], "params": {"weight": [[0.294921875]]}}, "language_model.model.layers.46.self_attn.qkv_proj": {"inputs": [[[26.75]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.46.self_attn.o_proj": {"inputs": [[[14.375]]], "outputs": [[[101.0]], [[188.0]]], "params": {"weight": [[0.138671875]]}}, "language_model.model.layers.46.self_attn.attn.impl.matmul_qk": {"inputs": [[[1.5546875]], [[17.75]]]}, "language_model.model.layers.46.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[14.5]]]}, "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[1.5546875]]]}, "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[11.9375]]]}, "language_model.model.layers.46.self_attn.attn.impl.k_cache": {"inputs": [[[17.75]]]}, "language_model.model.layers.46.self_attn.attn.impl.v_cache": {"inputs": [[[14.5]]]}, "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[18.5]], [[17.75]], [[14.5]]], "outputs": [[[14.375]], [[1.0]]]}, "language_model.model.layers.46.feed_forward.gate_up_proj": {"inputs": [[[40.5]]], "params": {"weight": [[0.63671875]]}}, "language_model.model.layers.46.feed_forward.down_proj": {"inputs": [[[2496.0]]], "outputs": [[[1688.0]], [[2400.0]]], "params": {"weight": [[0.51171875]]}}, "language_model.model.layers.47.self_attn.qkv_proj": {"inputs": [[[19.875]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.47.self_attn.o_proj": {"inputs": [[[7.8125]]], "outputs": [[[98.0]], [[126.5]]], "params": {"weight": [[0.7421875]]}}, "language_model.model.layers.47.self_attn.attn.impl.matmul_qk": {"inputs": [[[0.53515625]], [[14.375]]]}, "language_model.model.layers.47.self_attn.attn.impl.matmul_av": {"inputs": [[[1.0]], [[9.5625]]]}, "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul": {"inputs": [[[1.0]], [[0.53515625]]]}, "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul": {"inputs": [[[1.0]], [[6.75]]]}, "language_model.model.layers.47.self_attn.attn.impl.k_cache": {"inputs": [[[14.375]]]}, "language_model.model.layers.47.self_attn.attn.impl.v_cache": {"inputs": [[[9.5625]]]}, "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention": {"inputs": [[[6.125]], [[14.375]], [[9.5625]]], "outputs": [[[7.8125]], [[1.0]]]}, "language_model.model.layers.47.feed_forward.router": {"inputs": [[[33.5]]], "params": {"weight": [[0.37890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op": {"inputs": [[[33.5]]], "outputs": [[[162.0]], [[66.0]], [[102.0]], [[109.5]], [[89.0]], [[79.5]], [[32.25]], [[59.75]], [[57.5]], [[102.5]], [[219.0]], [[65.0]], [[121.5]], [[85.5]], [[42.25]], [[137.0]], [[50.5]], [[44.75]], [[47.0]], [[54.75]], [[59.0]], [[98.0]], [[96.5]], [[50.25]], [[72.0]], [[150.0]], [[129.0]], [[31.125]], [[28.75]], [[52.0]], [[42.5]], [[42.25]], [[84.0]], [[32.75]], [[48.5]], [[133.0]], [[35.5]], [[40.25]], [[22.875]], [[256.0]], [[100.5]], [[35.0]], [[137.0]], [[112.0]], [[27.375]], [[40.25]], [[74.5]], [[148.0]], [[67.5]], [[27.875]], [[127.0]], [[284.0]], [[310.0]], [[33.25]], [[25.25]], [[43.5]], [[41.0]], [[147.0]], [[33.25]], [[34.75]], [[192.0]], [[61.0]], [[94.0]], [[85.0]], [[124.0]], [[64.0]], [[29.0]], [[142.0]], [[123.5]], [[24.125]], [[258.0]], [[230.0]], [[26.5]], [[68.5]], [[57.5]], [[53.0]], [[25.125]], [[52.75]], [[102.0]], [[230.0]], [[27.375]], [[84.5]], [[230.0]], [[760.0]], [[75.0]], [[33.75]], [[68.0]], [[148.0]], [[48.25]], [[29.75]], [[108.5]], [[124.5]], [[141.0]], [[148.0]], [[34.75]], [[43.0]], [[123.0]], [[109.0]], [[35.25]], [[24.5]], [[50.0]], [[54.75]], [[70.5]], [[75.5]], [[31.25]], [[84.0]], [[47.5]], [[49.0]], [[288.0]], [[45.5]], [[52.0]], [[122.0]], [[195.0]], [[47.75]], [[65.0]], [[66.0]], [[58.75]], [[44.25]], [[178.0]], [[148.0]], [[282.0]], [[148.0]], [[70.0]], [[148.0]], [[24.875]], [[38.25]], [[134.0]], [[157.0]], [[219.0]]]}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.20703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.1162109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.10400390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.17578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.2373046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.11962890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.22265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.14453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.1826171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.0869140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.34765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.13671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.11865234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.0947265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.087890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.1787109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.09130859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.11376953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.1494140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.337890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.111328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.146484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.1845703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.10009765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.1533203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.1240234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.10498046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.1552734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.099609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.1748046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.0966796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.1611328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.10986328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[0.1455078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.08984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.119140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.11767578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.09033203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.1337890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.091796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.169921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.1435546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.09765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.09375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.142578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.2021484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.10107421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.095703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.10791015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1064453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.09716796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.1865234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.11474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.185546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.09521484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.09814453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.09912109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.08740234375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.0927734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.0888671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.0908203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.1025390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.154296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.12353515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.18359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.1640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.2001953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0": {"inputs": [[[0.0]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1": {"inputs": [[[0.0]]], "params": {"weight": [[0.1962890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2": {"inputs": [[[0.0]]], "params": {"weight": [[0.474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3": {"inputs": [[[0.0]]], "params": {"weight": [[0.765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4": {"inputs": [[[0.0]]], "params": {"weight": [[0.435546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5": {"inputs": [[[0.0]]], "params": {"weight": [[0.1318359375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6": {"inputs": [[[0.0]]], "params": {"weight": [[0.2158203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7": {"inputs": [[[0.0]]], "params": {"weight": [[0.2470703125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8": {"inputs": [[[0.0]]], "params": {"weight": [[0.75390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10": {"inputs": [[[0.0]]], "params": {"weight": [[0.39453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11": {"inputs": [[[0.0]]], "params": {"weight": [[0.73046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12": {"inputs": [[[0.0]]], "params": {"weight": [[0.5546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13": {"inputs": [[[0.0]]], "params": {"weight": [[0.09228515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14": {"inputs": [[[0.0]]], "params": {"weight": [[0.259765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15": {"inputs": [[[0.0]]], "params": {"weight": [[0.123046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.16": {"inputs": [[[0.0]]], "params": {"weight": [[0.28125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.17": {"inputs": [[[0.0]]], "params": {"weight": [[0.306640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.18": {"inputs": [[[0.0]]], "params": {"weight": [[0.91796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.19": {"inputs": [[[0.0]]], "params": {"weight": [[0.44921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.20": {"inputs": [[[0.0]]], "params": {"weight": [[0.9140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.21": {"inputs": [[[0.0]]], "params": {"weight": [[0.2119140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.22": {"inputs": [[[0.0]]], "params": {"weight": [[0.419921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.23": {"inputs": [[[0.0]]], "params": {"weight": [[0.2080078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.24": {"inputs": [[[0.0]]], "params": {"weight": [[0.439453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.25": {"inputs": [[[0.0]]], "params": {"weight": [[0.37109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.26": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.27": {"inputs": [[[0.0]]], "params": {"weight": [[0.251953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.28": {"inputs": [[[0.0]]], "params": {"weight": [[0.6015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.29": {"inputs": [[[0.0]]], "params": {"weight": [[0.11181640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.30": {"inputs": [[[0.0]]], "params": {"weight": [[1.046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.31": {"inputs": [[[0.0]]], "params": {"weight": [[0.462890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.32": {"inputs": [[[0.0]]], "params": {"weight": [[0.80078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.33": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.34": {"inputs": [[[0.0]]], "params": {"weight": [[0.76171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.35": {"inputs": [[[0.0]]], "params": {"weight": [[0.189453125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.36": {"inputs": [[[0.0]]], "params": {"weight": [[0.4609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.37": {"inputs": [[[0.0]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.38": {"inputs": [[[0.0]]], "params": {"weight": [[0.61328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.39": {"inputs": [[[0.0]]], "params": {"weight": [[0.75390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.40": {"inputs": [[[0.0]]], "params": {"weight": [[0.30078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.41": {"inputs": [[[0.0]]], "params": {"weight": [[0.52734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.42": {"inputs": [[[0.0]]], "params": {"weight": [[0.546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.43": {"inputs": [[[0.0]]], "params": {"weight": [[0.26171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.44": {"inputs": [[[0.0]]], "params": {"weight": [[0.71875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.45": {"inputs": [[[0.0]]], "params": {"weight": [[0.34765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.46": {"inputs": [[[0.0]]], "params": {"weight": [[0.23828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.47": {"inputs": [[[0.0]]], "params": {"weight": [[0.46484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.48": {"inputs": [[[0.0]]], "params": {"weight": [[0.09326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.49": {"inputs": [[[0.0]]], "params": {"weight": [[0.197265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.50": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.51": {"inputs": [[[0.0]]], "params": {"weight": [[0.59375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.52": {"inputs": [[[0.0]]], "params": {"weight": [[0.1416015625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.53": {"inputs": [[[0.0]]], "params": {"weight": [[0.27734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.54": {"inputs": [[[0.0]]], "params": {"weight": [[0.11669921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.55": {"inputs": [[[0.0]]], "params": {"weight": [[0.369140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.56": {"inputs": [[[0.0]]], "params": {"weight": [[0.466796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.57": {"inputs": [[[0.0]]], "params": {"weight": [[0.162109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.58": {"inputs": [[[0.0]]], "params": {"weight": [[0.1767578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.59": {"inputs": [[[0.0]]], "params": {"weight": [[0.21875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.60": {"inputs": [[[0.0]]], "params": {"weight": [[0.2392578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.61": {"inputs": [[[0.0]]], "params": {"weight": [[0.73828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.62": {"inputs": [[[0.0]]], "params": {"weight": [[0.82421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.63": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.64": {"inputs": [[[0.0]]], "params": {"weight": [[0.7578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.65": {"inputs": [[[0.0]]], "params": {"weight": [[0.478515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.66": {"inputs": [[[0.0]]], "params": {"weight": [[0.5078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.67": {"inputs": [[[0.0]]], "params": {"weight": [[0.3125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.68": {"inputs": [[[0.0]]], "params": {"weight": [[0.25390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.69": {"inputs": [[[0.0]]], "params": {"weight": [[0.734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.70": {"inputs": [[[0.0]]], "params": {"weight": [[1.0546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.71": {"inputs": [[[0.0]]], "params": {"weight": [[0.11279296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.72": {"inputs": [[[0.0]]], "params": {"weight": [[0.236328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.73": {"inputs": [[[0.0]]], "params": {"weight": [[0.205078125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.74": {"inputs": [[[0.0]]], "params": {"weight": [[0.3828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.75": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.76": {"inputs": [[[0.0]]], "params": {"weight": [[0.1728515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.77": {"inputs": [[[0.0]]], "params": {"weight": [[0.10302734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.78": {"inputs": [[[0.0]]], "params": {"weight": [[0.4140625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.79": {"inputs": [[[0.0]]], "params": {"weight": [[0.1708984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.80": {"inputs": [[[0.0]]], "params": {"weight": [[0.349609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.81": {"inputs": [[[0.0]]], "params": {"weight": [[0.29296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.82": {"inputs": [[[0.0]]], "params": {"weight": [[0.67578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.83": {"inputs": [[[0.0]]], "params": {"weight": [[0.77734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.84": {"inputs": [[[0.0]]], "params": {"weight": [[0.1298828125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.85": {"inputs": [[[0.0]]], "params": {"weight": [[0.515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.86": {"inputs": [[[0.0]]], "params": {"weight": [[0.326171875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.87": {"inputs": [[[0.0]]], "params": {"weight": [[0.2197265625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.88": {"inputs": [[[0.0]]], "params": {"weight": [[0.4296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.89": {"inputs": [[[0.0]]], "params": {"weight": [[0.78125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.90": {"inputs": [[[0.0]]], "params": {"weight": [[0.69921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.91": {"inputs": [[[0.0]]], "params": {"weight": [[0.2890625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.92": {"inputs": [[[0.0]]], "params": {"weight": [[0.388671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.93": {"inputs": [[[0.0]]], "params": {"weight": [[0.35546875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.94": {"inputs": [[[0.0]]], "params": {"weight": [[0.1142578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.95": {"inputs": [[[0.0]]], "params": {"weight": [[0.486328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.96": {"inputs": [[[0.0]]], "params": {"weight": [[0.7421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.97": {"inputs": [[[0.0]]], "params": {"weight": [[0.3984375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.98": {"inputs": [[[0.0]]], "params": {"weight": [[0.283203125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.99": {"inputs": [[[0.0]]], "params": {"weight": [[0.65625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.100": {"inputs": [[[0.0]]], "params": {"weight": [[0.19921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.101": {"inputs": [[[0.0]]], "params": {"weight": [[0.1669921875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.102": {"inputs": [[[0.0]]], "params": {"weight": [[0.671875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.103": {"inputs": [[[0.0]]], "params": {"weight": [[0.1630859375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.104": {"inputs": [[[0.0]]], "params": {"weight": [[0.59765625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.105": {"inputs": [[[0.0]]], "params": {"weight": [[0.2431640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.106": {"inputs": [[[0.0]]], "params": {"weight": [[0.357421875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.107": {"inputs": [[[0.0]]], "params": {"weight": [[0.5]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.108": {"inputs": [[[0.0]]], "params": {"weight": [[0.31640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.109": {"inputs": [[[0.0]]], "params": {"weight": [[0.96875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.110": {"inputs": [[[0.0]]], "params": {"weight": [[0.2353515625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.111": {"inputs": [[[0.0]]], "params": {"weight": [[0.61328125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.112": {"inputs": [[[0.0]]], "params": {"weight": [[0.1474609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.113": {"inputs": [[[0.0]]], "params": {"weight": [[0.2216796875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.114": {"inputs": [[[0.0]]], "params": {"weight": [[0.1875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.115": {"inputs": [[[0.0]]], "params": {"weight": [[0.287109375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.116": {"inputs": [[[0.0]]], "params": {"weight": [[0.1376953125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.117": {"inputs": [[[0.0]]], "params": {"weight": [[0.578125]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.118": {"inputs": [[[0.0]]], "params": {"weight": [[0.275390625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.119": {"inputs": [[[0.0]]], "params": {"weight": [[0.73046875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.120": {"inputs": [[[0.0]]], "params": {"weight": [[0.404296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.121": {"inputs": [[[0.0]]], "params": {"weight": [[0.4609375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.122": {"inputs": [[[0.0]]], "params": {"weight": [[0.6640625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.123": {"inputs": [[[0.0]]], "params": {"weight": [[0.15625]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.124": {"inputs": [[[0.0]]], "params": {"weight": [[0.7734375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.125": {"inputs": [[[0.0]]], "params": {"weight": [[0.79296875]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.126": {"inputs": [[[0.0]]], "params": {"weight": [[0.396484375]]}}, "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.127": {"inputs": [[[0.0]]], "params": {"weight": [[0.5390625]]}}, "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj": {"inputs": [[[33.5]]], "params": {"weight": [[0.56640625]]}}, "language_model.model.layers.47.feed_forward.shared_expert.down_proj": {"inputs": [[[1168.0]]], "outputs": [[[564.0]], [[1.6096120261457966e+35]]], "params": {"weight": [[0.45703125]]}}, "language_model.lm_head": {"inputs": [[[52.0]]], "params": {"weight": [[1.0703125]]}}}} \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_7_8.npz b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_7_8.npz new file mode 100644 index 000000000000..cbe2019399d0 Binary files /dev/null and b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_7_8.npz differ diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_7_8_mod_list.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_7_8_mod_list.json new file mode 100644 index 000000000000..928cd9e2a2c5 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output_hooks_maxabs_7_8_mod_list.json @@ -0,0 +1,6935 @@ +[ + "vision_model.patch_embedding.linear", + "vision_model.model.layers.0.self_attn.qkv_proj", + "vision_model.model.layers.0.self_attn.o_proj", + "vision_model.model.layers.0.mlp.fc1", + "vision_model.model.layers.0.mlp.fc2", + "vision_model.model.layers.1.self_attn.qkv_proj", + "vision_model.model.layers.1.self_attn.o_proj", + "vision_model.model.layers.1.mlp.fc1", + "vision_model.model.layers.1.mlp.fc2", + "vision_model.model.layers.2.self_attn.qkv_proj", + "vision_model.model.layers.2.self_attn.o_proj", + "vision_model.model.layers.2.mlp.fc1", + "vision_model.model.layers.2.mlp.fc2", + "vision_model.model.layers.3.self_attn.qkv_proj", + "vision_model.model.layers.3.self_attn.o_proj", + "vision_model.model.layers.3.mlp.fc1", + "vision_model.model.layers.3.mlp.fc2", + "vision_model.model.layers.4.self_attn.qkv_proj", + "vision_model.model.layers.4.self_attn.o_proj", + "vision_model.model.layers.4.mlp.fc1", + "vision_model.model.layers.4.mlp.fc2", + "vision_model.model.layers.5.self_attn.qkv_proj", + "vision_model.model.layers.5.self_attn.o_proj", + "vision_model.model.layers.5.mlp.fc1", + "vision_model.model.layers.5.mlp.fc2", + "vision_model.model.layers.6.self_attn.qkv_proj", + "vision_model.model.layers.6.self_attn.o_proj", + "vision_model.model.layers.6.mlp.fc1", + "vision_model.model.layers.6.mlp.fc2", + "vision_model.model.layers.7.self_attn.qkv_proj", + "vision_model.model.layers.7.self_attn.o_proj", + "vision_model.model.layers.7.mlp.fc1", + "vision_model.model.layers.7.mlp.fc2", + "vision_model.model.layers.8.self_attn.qkv_proj", + "vision_model.model.layers.8.self_attn.o_proj", + "vision_model.model.layers.8.mlp.fc1", + "vision_model.model.layers.8.mlp.fc2", + "vision_model.model.layers.9.self_attn.qkv_proj", + "vision_model.model.layers.9.self_attn.o_proj", + "vision_model.model.layers.9.mlp.fc1", + "vision_model.model.layers.9.mlp.fc2", + "vision_model.model.layers.10.self_attn.qkv_proj", + "vision_model.model.layers.10.self_attn.o_proj", + "vision_model.model.layers.10.mlp.fc1", + "vision_model.model.layers.10.mlp.fc2", + "vision_model.model.layers.11.self_attn.qkv_proj", + "vision_model.model.layers.11.self_attn.o_proj", + "vision_model.model.layers.11.mlp.fc1", + "vision_model.model.layers.11.mlp.fc2", + "vision_model.model.layers.12.self_attn.qkv_proj", + "vision_model.model.layers.12.self_attn.o_proj", + "vision_model.model.layers.12.mlp.fc1", + "vision_model.model.layers.12.mlp.fc2", + "vision_model.model.layers.13.self_attn.qkv_proj", + "vision_model.model.layers.13.self_attn.o_proj", + "vision_model.model.layers.13.mlp.fc1", + "vision_model.model.layers.13.mlp.fc2", + "vision_model.model.layers.14.self_attn.qkv_proj", + "vision_model.model.layers.14.self_attn.o_proj", + "vision_model.model.layers.14.mlp.fc1", + "vision_model.model.layers.14.mlp.fc2", + "vision_model.model.layers.15.self_attn.qkv_proj", + "vision_model.model.layers.15.self_attn.o_proj", + "vision_model.model.layers.15.mlp.fc1", + "vision_model.model.layers.15.mlp.fc2", + "vision_model.model.layers.16.self_attn.qkv_proj", + "vision_model.model.layers.16.self_attn.o_proj", + "vision_model.model.layers.16.mlp.fc1", + "vision_model.model.layers.16.mlp.fc2", + "vision_model.model.layers.17.self_attn.qkv_proj", + "vision_model.model.layers.17.self_attn.o_proj", + "vision_model.model.layers.17.mlp.fc1", + "vision_model.model.layers.17.mlp.fc2", + "vision_model.model.layers.18.self_attn.qkv_proj", + "vision_model.model.layers.18.self_attn.o_proj", + "vision_model.model.layers.18.mlp.fc1", + "vision_model.model.layers.18.mlp.fc2", + "vision_model.model.layers.19.self_attn.qkv_proj", + "vision_model.model.layers.19.self_attn.o_proj", + "vision_model.model.layers.19.mlp.fc1", + "vision_model.model.layers.19.mlp.fc2", + "vision_model.model.layers.20.self_attn.qkv_proj", + "vision_model.model.layers.20.self_attn.o_proj", + "vision_model.model.layers.20.mlp.fc1", + "vision_model.model.layers.20.mlp.fc2", + "vision_model.model.layers.21.self_attn.qkv_proj", + "vision_model.model.layers.21.self_attn.o_proj", + "vision_model.model.layers.21.mlp.fc1", + "vision_model.model.layers.21.mlp.fc2", + "vision_model.model.layers.22.self_attn.qkv_proj", + "vision_model.model.layers.22.self_attn.o_proj", + "vision_model.model.layers.22.mlp.fc1", + "vision_model.model.layers.22.mlp.fc2", + "vision_model.model.layers.23.self_attn.qkv_proj", + "vision_model.model.layers.23.self_attn.o_proj", + "vision_model.model.layers.23.mlp.fc1", + "vision_model.model.layers.23.mlp.fc2", + "vision_model.model.layers.24.self_attn.qkv_proj", + "vision_model.model.layers.24.self_attn.o_proj", + "vision_model.model.layers.24.mlp.fc1", + "vision_model.model.layers.24.mlp.fc2", + "vision_model.model.layers.25.self_attn.qkv_proj", + "vision_model.model.layers.25.self_attn.o_proj", + "vision_model.model.layers.25.mlp.fc1", + "vision_model.model.layers.25.mlp.fc2", + "vision_model.model.layers.26.self_attn.qkv_proj", + "vision_model.model.layers.26.self_attn.o_proj", + "vision_model.model.layers.26.mlp.fc1", + "vision_model.model.layers.26.mlp.fc2", + "vision_model.model.layers.27.self_attn.qkv_proj", + "vision_model.model.layers.27.self_attn.o_proj", + "vision_model.model.layers.27.mlp.fc1", + "vision_model.model.layers.27.mlp.fc2", + "vision_model.model.layers.28.self_attn.qkv_proj", + "vision_model.model.layers.28.self_attn.o_proj", + "vision_model.model.layers.28.mlp.fc1", + "vision_model.model.layers.28.mlp.fc2", + "vision_model.model.layers.29.self_attn.qkv_proj", + "vision_model.model.layers.29.self_attn.o_proj", + "vision_model.model.layers.29.mlp.fc1", + "vision_model.model.layers.29.mlp.fc2", + "vision_model.model.layers.30.self_attn.qkv_proj", + "vision_model.model.layers.30.self_attn.o_proj", + "vision_model.model.layers.30.mlp.fc1", + "vision_model.model.layers.30.mlp.fc2", + "vision_model.model.layers.31.self_attn.qkv_proj", + "vision_model.model.layers.31.self_attn.o_proj", + "vision_model.model.layers.31.mlp.fc1", + "vision_model.model.layers.31.mlp.fc2", + "vision_model.model.layers.32.self_attn.qkv_proj", + "vision_model.model.layers.32.self_attn.o_proj", + "vision_model.model.layers.32.mlp.fc1", + "vision_model.model.layers.32.mlp.fc2", + "vision_model.model.layers.33.self_attn.qkv_proj", + "vision_model.model.layers.33.self_attn.o_proj", + "vision_model.model.layers.33.mlp.fc1", + "vision_model.model.layers.33.mlp.fc2", + "vision_model.vision_adapter.mlp.fc1", + "vision_model.vision_adapter.mlp.fc2", + "multi_modal_projector.linear_1", + "language_model.model.layers.0.self_attn.qkv_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.0.self_attn.attn.impl.softmax", + "language_model.model.layers.0.self_attn.attn.impl.matmul_av", + "language_model.model.layers.0.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.0.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.0.self_attn.attn.impl.k_cache", + "language_model.model.layers.0.self_attn.attn.impl.v_cache", + "language_model.model.layers.0.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.0.feed_forward.gate_up_proj", + "language_model.model.layers.0.feed_forward.down_proj", + "language_model.model.layers.1.self_attn.qkv_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.1.self_attn.attn.impl.softmax", + "language_model.model.layers.1.self_attn.attn.impl.matmul_av", + "language_model.model.layers.1.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.1.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.1.self_attn.attn.impl.k_cache", + "language_model.model.layers.1.self_attn.attn.impl.v_cache", + "language_model.model.layers.1.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.1.feed_forward.router", + "language_model.model.layers.1.feed_forward.experts", + "language_model.model.layers.1.feed_forward.experts.moe_op", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.1.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.1.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.1.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.1.feed_forward.shared_expert.down_proj", + "language_model.model.layers.2.self_attn.qkv_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.2.self_attn.attn.impl.softmax", + "language_model.model.layers.2.self_attn.attn.impl.matmul_av", + "language_model.model.layers.2.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.2.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.2.self_attn.attn.impl.k_cache", + "language_model.model.layers.2.self_attn.attn.impl.v_cache", + "language_model.model.layers.2.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.2.feed_forward.gate_up_proj", + "language_model.model.layers.2.feed_forward.down_proj", + "language_model.model.layers.3.self_attn.qkv_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.3.self_attn.attn.impl.softmax", + "language_model.model.layers.3.self_attn.attn.impl.matmul_av", + "language_model.model.layers.3.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.3.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.3.self_attn.attn.impl.k_cache", + "language_model.model.layers.3.self_attn.attn.impl.v_cache", + "language_model.model.layers.3.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.3.feed_forward.router", + "language_model.model.layers.3.feed_forward.experts", + "language_model.model.layers.3.feed_forward.experts.moe_op", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.3.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.3.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.3.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.3.feed_forward.shared_expert.down_proj", + "language_model.model.layers.4.self_attn.qkv_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.4.self_attn.attn.impl.softmax", + "language_model.model.layers.4.self_attn.attn.impl.matmul_av", + "language_model.model.layers.4.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.4.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.4.self_attn.attn.impl.k_cache", + "language_model.model.layers.4.self_attn.attn.impl.v_cache", + "language_model.model.layers.4.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.4.feed_forward.gate_up_proj", + "language_model.model.layers.4.feed_forward.down_proj", + "language_model.model.layers.5.self_attn.qkv_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.5.self_attn.attn.impl.softmax", + "language_model.model.layers.5.self_attn.attn.impl.matmul_av", + "language_model.model.layers.5.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.5.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.5.self_attn.attn.impl.k_cache", + "language_model.model.layers.5.self_attn.attn.impl.v_cache", + "language_model.model.layers.5.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.5.feed_forward.router", + "language_model.model.layers.5.feed_forward.experts", + "language_model.model.layers.5.feed_forward.experts.moe_op", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.5.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.5.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.5.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.5.feed_forward.shared_expert.down_proj", + "language_model.model.layers.6.self_attn.qkv_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.6.self_attn.attn.impl.softmax", + "language_model.model.layers.6.self_attn.attn.impl.matmul_av", + "language_model.model.layers.6.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.6.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.6.self_attn.attn.impl.k_cache", + "language_model.model.layers.6.self_attn.attn.impl.v_cache", + "language_model.model.layers.6.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.6.feed_forward.gate_up_proj", + "language_model.model.layers.6.feed_forward.down_proj", + "language_model.model.layers.7.self_attn.qkv_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.7.self_attn.attn.impl.softmax", + "language_model.model.layers.7.self_attn.attn.impl.matmul_av", + "language_model.model.layers.7.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.7.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.7.self_attn.attn.impl.k_cache", + "language_model.model.layers.7.self_attn.attn.impl.v_cache", + "language_model.model.layers.7.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.7.feed_forward.router", + "language_model.model.layers.7.feed_forward.experts", + "language_model.model.layers.7.feed_forward.experts.moe_op", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.7.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.7.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.7.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.7.feed_forward.shared_expert.down_proj", + "language_model.model.layers.8.self_attn.qkv_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.8.self_attn.attn.impl.softmax", + "language_model.model.layers.8.self_attn.attn.impl.matmul_av", + "language_model.model.layers.8.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.8.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.8.self_attn.attn.impl.k_cache", + "language_model.model.layers.8.self_attn.attn.impl.v_cache", + "language_model.model.layers.8.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.8.feed_forward.gate_up_proj", + "language_model.model.layers.8.feed_forward.down_proj", + "language_model.model.layers.9.self_attn.qkv_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.9.self_attn.attn.impl.softmax", + "language_model.model.layers.9.self_attn.attn.impl.matmul_av", + "language_model.model.layers.9.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.9.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.9.self_attn.attn.impl.k_cache", + "language_model.model.layers.9.self_attn.attn.impl.v_cache", + "language_model.model.layers.9.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.9.feed_forward.router", + "language_model.model.layers.9.feed_forward.experts", + "language_model.model.layers.9.feed_forward.experts.moe_op", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.9.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.9.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.9.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.9.feed_forward.shared_expert.down_proj", + "language_model.model.layers.10.self_attn.qkv_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.10.self_attn.attn.impl.softmax", + "language_model.model.layers.10.self_attn.attn.impl.matmul_av", + "language_model.model.layers.10.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.10.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.10.self_attn.attn.impl.k_cache", + "language_model.model.layers.10.self_attn.attn.impl.v_cache", + "language_model.model.layers.10.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.10.feed_forward.gate_up_proj", + "language_model.model.layers.10.feed_forward.down_proj", + "language_model.model.layers.11.self_attn.qkv_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.11.self_attn.attn.impl.softmax", + "language_model.model.layers.11.self_attn.attn.impl.matmul_av", + "language_model.model.layers.11.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.11.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.11.self_attn.attn.impl.k_cache", + "language_model.model.layers.11.self_attn.attn.impl.v_cache", + "language_model.model.layers.11.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.11.feed_forward.router", + "language_model.model.layers.11.feed_forward.experts", + "language_model.model.layers.11.feed_forward.experts.moe_op", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.11.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.11.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.11.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.11.feed_forward.shared_expert.down_proj", + "language_model.model.layers.12.self_attn.qkv_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.12.self_attn.attn.impl.softmax", + "language_model.model.layers.12.self_attn.attn.impl.matmul_av", + "language_model.model.layers.12.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.12.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.12.self_attn.attn.impl.k_cache", + "language_model.model.layers.12.self_attn.attn.impl.v_cache", + "language_model.model.layers.12.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.12.feed_forward.gate_up_proj", + "language_model.model.layers.12.feed_forward.down_proj", + "language_model.model.layers.13.self_attn.qkv_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.13.self_attn.attn.impl.softmax", + "language_model.model.layers.13.self_attn.attn.impl.matmul_av", + "language_model.model.layers.13.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.13.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.13.self_attn.attn.impl.k_cache", + "language_model.model.layers.13.self_attn.attn.impl.v_cache", + "language_model.model.layers.13.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.13.feed_forward.router", + "language_model.model.layers.13.feed_forward.experts", + "language_model.model.layers.13.feed_forward.experts.moe_op", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.13.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.13.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.13.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.13.feed_forward.shared_expert.down_proj", + "language_model.model.layers.14.self_attn.qkv_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.14.self_attn.attn.impl.softmax", + "language_model.model.layers.14.self_attn.attn.impl.matmul_av", + "language_model.model.layers.14.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.14.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.14.self_attn.attn.impl.k_cache", + "language_model.model.layers.14.self_attn.attn.impl.v_cache", + "language_model.model.layers.14.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.14.feed_forward.gate_up_proj", + "language_model.model.layers.14.feed_forward.down_proj", + "language_model.model.layers.15.self_attn.qkv_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.15.self_attn.attn.impl.softmax", + "language_model.model.layers.15.self_attn.attn.impl.matmul_av", + "language_model.model.layers.15.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.15.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.15.self_attn.attn.impl.k_cache", + "language_model.model.layers.15.self_attn.attn.impl.v_cache", + "language_model.model.layers.15.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.15.feed_forward.router", + "language_model.model.layers.15.feed_forward.experts", + "language_model.model.layers.15.feed_forward.experts.moe_op", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.15.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.15.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.15.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.15.feed_forward.shared_expert.down_proj", + "language_model.model.layers.16.self_attn.qkv_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.16.self_attn.attn.impl.softmax", + "language_model.model.layers.16.self_attn.attn.impl.matmul_av", + "language_model.model.layers.16.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.16.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.16.self_attn.attn.impl.k_cache", + "language_model.model.layers.16.self_attn.attn.impl.v_cache", + "language_model.model.layers.16.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.16.feed_forward.gate_up_proj", + "language_model.model.layers.16.feed_forward.down_proj", + "language_model.model.layers.17.self_attn.qkv_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.17.self_attn.attn.impl.softmax", + "language_model.model.layers.17.self_attn.attn.impl.matmul_av", + "language_model.model.layers.17.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.17.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.17.self_attn.attn.impl.k_cache", + "language_model.model.layers.17.self_attn.attn.impl.v_cache", + "language_model.model.layers.17.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.17.feed_forward.router", + "language_model.model.layers.17.feed_forward.experts", + "language_model.model.layers.17.feed_forward.experts.moe_op", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.17.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.17.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.17.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.17.feed_forward.shared_expert.down_proj", + "language_model.model.layers.18.self_attn.qkv_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.18.self_attn.attn.impl.softmax", + "language_model.model.layers.18.self_attn.attn.impl.matmul_av", + "language_model.model.layers.18.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.18.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.18.self_attn.attn.impl.k_cache", + "language_model.model.layers.18.self_attn.attn.impl.v_cache", + "language_model.model.layers.18.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.18.feed_forward.gate_up_proj", + "language_model.model.layers.18.feed_forward.down_proj", + "language_model.model.layers.19.self_attn.qkv_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.19.self_attn.attn.impl.softmax", + "language_model.model.layers.19.self_attn.attn.impl.matmul_av", + "language_model.model.layers.19.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.19.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.19.self_attn.attn.impl.k_cache", + "language_model.model.layers.19.self_attn.attn.impl.v_cache", + "language_model.model.layers.19.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.19.feed_forward.router", + "language_model.model.layers.19.feed_forward.experts", + "language_model.model.layers.19.feed_forward.experts.moe_op", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.19.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.19.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.19.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.19.feed_forward.shared_expert.down_proj", + "language_model.model.layers.20.self_attn.qkv_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.20.self_attn.attn.impl.softmax", + "language_model.model.layers.20.self_attn.attn.impl.matmul_av", + "language_model.model.layers.20.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.20.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.20.self_attn.attn.impl.k_cache", + "language_model.model.layers.20.self_attn.attn.impl.v_cache", + "language_model.model.layers.20.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.20.feed_forward.gate_up_proj", + "language_model.model.layers.20.feed_forward.down_proj", + "language_model.model.layers.21.self_attn.qkv_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.21.self_attn.attn.impl.softmax", + "language_model.model.layers.21.self_attn.attn.impl.matmul_av", + "language_model.model.layers.21.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.21.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.21.self_attn.attn.impl.k_cache", + "language_model.model.layers.21.self_attn.attn.impl.v_cache", + "language_model.model.layers.21.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.21.feed_forward.router", + "language_model.model.layers.21.feed_forward.experts", + "language_model.model.layers.21.feed_forward.experts.moe_op", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.21.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.21.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.21.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.21.feed_forward.shared_expert.down_proj", + "language_model.model.layers.22.self_attn.qkv_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.22.self_attn.attn.impl.softmax", + "language_model.model.layers.22.self_attn.attn.impl.matmul_av", + "language_model.model.layers.22.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.22.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.22.self_attn.attn.impl.k_cache", + "language_model.model.layers.22.self_attn.attn.impl.v_cache", + "language_model.model.layers.22.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.22.feed_forward.gate_up_proj", + "language_model.model.layers.22.feed_forward.down_proj", + "language_model.model.layers.23.self_attn.qkv_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.23.self_attn.attn.impl.softmax", + "language_model.model.layers.23.self_attn.attn.impl.matmul_av", + "language_model.model.layers.23.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.23.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.23.self_attn.attn.impl.k_cache", + "language_model.model.layers.23.self_attn.attn.impl.v_cache", + "language_model.model.layers.23.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.23.feed_forward.router", + "language_model.model.layers.23.feed_forward.experts", + "language_model.model.layers.23.feed_forward.experts.moe_op", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.23.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.23.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.23.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.23.feed_forward.shared_expert.down_proj", + "language_model.model.layers.24.self_attn.qkv_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.24.self_attn.attn.impl.softmax", + "language_model.model.layers.24.self_attn.attn.impl.matmul_av", + "language_model.model.layers.24.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.24.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.24.self_attn.attn.impl.k_cache", + "language_model.model.layers.24.self_attn.attn.impl.v_cache", + "language_model.model.layers.24.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.24.feed_forward.gate_up_proj", + "language_model.model.layers.24.feed_forward.down_proj", + "language_model.model.layers.25.self_attn.qkv_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.25.self_attn.attn.impl.softmax", + "language_model.model.layers.25.self_attn.attn.impl.matmul_av", + "language_model.model.layers.25.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.25.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.25.self_attn.attn.impl.k_cache", + "language_model.model.layers.25.self_attn.attn.impl.v_cache", + "language_model.model.layers.25.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.25.feed_forward.router", + "language_model.model.layers.25.feed_forward.experts", + "language_model.model.layers.25.feed_forward.experts.moe_op", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.25.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.25.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.25.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.25.feed_forward.shared_expert.down_proj", + "language_model.model.layers.26.self_attn.qkv_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.26.self_attn.attn.impl.softmax", + "language_model.model.layers.26.self_attn.attn.impl.matmul_av", + "language_model.model.layers.26.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.26.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.26.self_attn.attn.impl.k_cache", + "language_model.model.layers.26.self_attn.attn.impl.v_cache", + "language_model.model.layers.26.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.26.feed_forward.gate_up_proj", + "language_model.model.layers.26.feed_forward.down_proj", + "language_model.model.layers.27.self_attn.qkv_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.27.self_attn.attn.impl.softmax", + "language_model.model.layers.27.self_attn.attn.impl.matmul_av", + "language_model.model.layers.27.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.27.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.27.self_attn.attn.impl.k_cache", + "language_model.model.layers.27.self_attn.attn.impl.v_cache", + "language_model.model.layers.27.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.27.feed_forward.router", + "language_model.model.layers.27.feed_forward.experts", + "language_model.model.layers.27.feed_forward.experts.moe_op", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.27.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.27.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.27.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.27.feed_forward.shared_expert.down_proj", + "language_model.model.layers.28.self_attn.qkv_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.28.self_attn.attn.impl.softmax", + "language_model.model.layers.28.self_attn.attn.impl.matmul_av", + "language_model.model.layers.28.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.28.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.28.self_attn.attn.impl.k_cache", + "language_model.model.layers.28.self_attn.attn.impl.v_cache", + "language_model.model.layers.28.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.28.feed_forward.gate_up_proj", + "language_model.model.layers.28.feed_forward.down_proj", + "language_model.model.layers.29.self_attn.qkv_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.29.self_attn.attn.impl.softmax", + "language_model.model.layers.29.self_attn.attn.impl.matmul_av", + "language_model.model.layers.29.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.29.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.29.self_attn.attn.impl.k_cache", + "language_model.model.layers.29.self_attn.attn.impl.v_cache", + "language_model.model.layers.29.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.29.feed_forward.router", + "language_model.model.layers.29.feed_forward.experts", + "language_model.model.layers.29.feed_forward.experts.moe_op", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.29.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.29.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.29.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.29.feed_forward.shared_expert.down_proj", + "language_model.model.layers.30.self_attn.qkv_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.30.self_attn.attn.impl.softmax", + "language_model.model.layers.30.self_attn.attn.impl.matmul_av", + "language_model.model.layers.30.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.30.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.30.self_attn.attn.impl.k_cache", + "language_model.model.layers.30.self_attn.attn.impl.v_cache", + "language_model.model.layers.30.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.30.feed_forward.gate_up_proj", + "language_model.model.layers.30.feed_forward.down_proj", + "language_model.model.layers.31.self_attn.qkv_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.31.self_attn.attn.impl.softmax", + "language_model.model.layers.31.self_attn.attn.impl.matmul_av", + "language_model.model.layers.31.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.31.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.31.self_attn.attn.impl.k_cache", + "language_model.model.layers.31.self_attn.attn.impl.v_cache", + "language_model.model.layers.31.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.31.feed_forward.router", + "language_model.model.layers.31.feed_forward.experts", + "language_model.model.layers.31.feed_forward.experts.moe_op", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.31.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.31.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.31.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.31.feed_forward.shared_expert.down_proj", + "language_model.model.layers.32.self_attn.qkv_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.32.self_attn.attn.impl.softmax", + "language_model.model.layers.32.self_attn.attn.impl.matmul_av", + "language_model.model.layers.32.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.32.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.32.self_attn.attn.impl.k_cache", + "language_model.model.layers.32.self_attn.attn.impl.v_cache", + "language_model.model.layers.32.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.32.feed_forward.gate_up_proj", + "language_model.model.layers.32.feed_forward.down_proj", + "language_model.model.layers.33.self_attn.qkv_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.33.self_attn.attn.impl.softmax", + "language_model.model.layers.33.self_attn.attn.impl.matmul_av", + "language_model.model.layers.33.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.33.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.33.self_attn.attn.impl.k_cache", + "language_model.model.layers.33.self_attn.attn.impl.v_cache", + "language_model.model.layers.33.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.33.feed_forward.router", + "language_model.model.layers.33.feed_forward.experts", + "language_model.model.layers.33.feed_forward.experts.moe_op", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.33.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.33.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.33.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.33.feed_forward.shared_expert.down_proj", + "language_model.model.layers.34.self_attn.qkv_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.34.self_attn.attn.impl.softmax", + "language_model.model.layers.34.self_attn.attn.impl.matmul_av", + "language_model.model.layers.34.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.34.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.34.self_attn.attn.impl.k_cache", + "language_model.model.layers.34.self_attn.attn.impl.v_cache", + "language_model.model.layers.34.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.34.feed_forward.gate_up_proj", + "language_model.model.layers.34.feed_forward.down_proj", + "language_model.model.layers.35.self_attn.qkv_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.35.self_attn.attn.impl.softmax", + "language_model.model.layers.35.self_attn.attn.impl.matmul_av", + "language_model.model.layers.35.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.35.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.35.self_attn.attn.impl.k_cache", + "language_model.model.layers.35.self_attn.attn.impl.v_cache", + "language_model.model.layers.35.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.35.feed_forward.router", + "language_model.model.layers.35.feed_forward.experts", + "language_model.model.layers.35.feed_forward.experts.moe_op", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.35.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.35.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.35.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.35.feed_forward.shared_expert.down_proj", + "language_model.model.layers.36.self_attn.qkv_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.36.self_attn.attn.impl.softmax", + "language_model.model.layers.36.self_attn.attn.impl.matmul_av", + "language_model.model.layers.36.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.36.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.36.self_attn.attn.impl.k_cache", + "language_model.model.layers.36.self_attn.attn.impl.v_cache", + "language_model.model.layers.36.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.36.feed_forward.gate_up_proj", + "language_model.model.layers.36.feed_forward.down_proj", + "language_model.model.layers.37.self_attn.qkv_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.37.self_attn.attn.impl.softmax", + "language_model.model.layers.37.self_attn.attn.impl.matmul_av", + "language_model.model.layers.37.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.37.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.37.self_attn.attn.impl.k_cache", + "language_model.model.layers.37.self_attn.attn.impl.v_cache", + "language_model.model.layers.37.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.37.feed_forward.router", + "language_model.model.layers.37.feed_forward.experts", + "language_model.model.layers.37.feed_forward.experts.moe_op", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.37.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.37.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.37.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.37.feed_forward.shared_expert.down_proj", + "language_model.model.layers.38.self_attn.qkv_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.38.self_attn.attn.impl.softmax", + "language_model.model.layers.38.self_attn.attn.impl.matmul_av", + "language_model.model.layers.38.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.38.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.38.self_attn.attn.impl.k_cache", + "language_model.model.layers.38.self_attn.attn.impl.v_cache", + "language_model.model.layers.38.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.38.feed_forward.gate_up_proj", + "language_model.model.layers.38.feed_forward.down_proj", + "language_model.model.layers.39.self_attn.qkv_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.39.self_attn.attn.impl.softmax", + "language_model.model.layers.39.self_attn.attn.impl.matmul_av", + "language_model.model.layers.39.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.39.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.39.self_attn.attn.impl.k_cache", + "language_model.model.layers.39.self_attn.attn.impl.v_cache", + "language_model.model.layers.39.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.39.feed_forward.router", + "language_model.model.layers.39.feed_forward.experts", + "language_model.model.layers.39.feed_forward.experts.moe_op", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.39.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.39.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.39.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.39.feed_forward.shared_expert.down_proj", + "language_model.model.layers.40.self_attn.qkv_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.40.self_attn.attn.impl.softmax", + "language_model.model.layers.40.self_attn.attn.impl.matmul_av", + "language_model.model.layers.40.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.40.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.40.self_attn.attn.impl.k_cache", + "language_model.model.layers.40.self_attn.attn.impl.v_cache", + "language_model.model.layers.40.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.40.feed_forward.gate_up_proj", + "language_model.model.layers.40.feed_forward.down_proj", + "language_model.model.layers.41.self_attn.qkv_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.41.self_attn.attn.impl.softmax", + "language_model.model.layers.41.self_attn.attn.impl.matmul_av", + "language_model.model.layers.41.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.41.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.41.self_attn.attn.impl.k_cache", + "language_model.model.layers.41.self_attn.attn.impl.v_cache", + "language_model.model.layers.41.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.41.feed_forward.router", + "language_model.model.layers.41.feed_forward.experts", + "language_model.model.layers.41.feed_forward.experts.moe_op", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.41.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.41.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.41.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.41.feed_forward.shared_expert.down_proj", + "language_model.model.layers.42.self_attn.qkv_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.42.self_attn.attn.impl.softmax", + "language_model.model.layers.42.self_attn.attn.impl.matmul_av", + "language_model.model.layers.42.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.42.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.42.self_attn.attn.impl.k_cache", + "language_model.model.layers.42.self_attn.attn.impl.v_cache", + "language_model.model.layers.42.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.42.feed_forward.gate_up_proj", + "language_model.model.layers.42.feed_forward.down_proj", + "language_model.model.layers.43.self_attn.qkv_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.43.self_attn.attn.impl.softmax", + "language_model.model.layers.43.self_attn.attn.impl.matmul_av", + "language_model.model.layers.43.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.43.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.43.self_attn.attn.impl.k_cache", + "language_model.model.layers.43.self_attn.attn.impl.v_cache", + "language_model.model.layers.43.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.43.feed_forward.router", + "language_model.model.layers.43.feed_forward.experts", + "language_model.model.layers.43.feed_forward.experts.moe_op", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.43.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.43.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.43.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.43.feed_forward.shared_expert.down_proj", + "language_model.model.layers.44.self_attn.qkv_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.44.self_attn.attn.impl.softmax", + "language_model.model.layers.44.self_attn.attn.impl.matmul_av", + "language_model.model.layers.44.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.44.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.44.self_attn.attn.impl.k_cache", + "language_model.model.layers.44.self_attn.attn.impl.v_cache", + "language_model.model.layers.44.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.44.feed_forward.gate_up_proj", + "language_model.model.layers.44.feed_forward.down_proj", + "language_model.model.layers.45.self_attn.qkv_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.45.self_attn.attn.impl.softmax", + "language_model.model.layers.45.self_attn.attn.impl.matmul_av", + "language_model.model.layers.45.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.45.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.45.self_attn.attn.impl.k_cache", + "language_model.model.layers.45.self_attn.attn.impl.v_cache", + "language_model.model.layers.45.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.45.feed_forward.router", + "language_model.model.layers.45.feed_forward.experts", + "language_model.model.layers.45.feed_forward.experts.moe_op", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.45.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.45.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.45.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.45.feed_forward.shared_expert.down_proj", + "language_model.model.layers.46.self_attn.qkv_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.46.self_attn.attn.impl.softmax", + "language_model.model.layers.46.self_attn.attn.impl.matmul_av", + "language_model.model.layers.46.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.46.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.46.self_attn.attn.impl.k_cache", + "language_model.model.layers.46.self_attn.attn.impl.v_cache", + "language_model.model.layers.46.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.46.feed_forward.gate_up_proj", + "language_model.model.layers.46.feed_forward.down_proj", + "language_model.model.layers.47.self_attn.qkv_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.attn.impl.matmul_qk", + "language_model.model.layers.47.self_attn.attn.impl.softmax", + "language_model.model.layers.47.self_attn.attn.impl.matmul_av", + "language_model.model.layers.47.self_attn.attn.impl.batch2block_matmul", + "language_model.model.layers.47.self_attn.attn.impl.block2batch_matmul", + "language_model.model.layers.47.self_attn.attn.impl.k_cache", + "language_model.model.layers.47.self_attn.attn.impl.v_cache", + "language_model.model.layers.47.self_attn.attn.impl.fused_scaled_dot_product_attention", + "language_model.model.layers.47.feed_forward.router", + "language_model.model.layers.47.feed_forward.experts", + "language_model.model.layers.47.feed_forward.experts.moe_op", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.16", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.17", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.18", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.19", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.20", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.21", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.22", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.23", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.24", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.25", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.26", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.27", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.28", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.29", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.30", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.31", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.32", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.33", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.34", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.35", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.36", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.37", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.38", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.39", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.40", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.41", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.42", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.43", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.44", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.45", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.46", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.47", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.48", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.49", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.50", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.51", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.52", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.53", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.54", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.55", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.56", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.57", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.58", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.59", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.60", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.61", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.62", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.63", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.64", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.65", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.66", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.67", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.68", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.69", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.70", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.71", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.72", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.73", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.74", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.75", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.76", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.77", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.78", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.79", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.80", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.81", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.82", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.83", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.84", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.85", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.86", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.87", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.88", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.89", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.90", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.91", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.92", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.93", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.94", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.95", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.96", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.97", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.98", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.99", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.100", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.101", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.102", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.103", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.104", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.105", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.106", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.107", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.108", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.109", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.110", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.111", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.112", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.113", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.114", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.115", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.116", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.117", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.118", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.119", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.120", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.121", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.122", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.123", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.124", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.125", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.126", + "language_model.model.layers.47.feed_forward.experts.moe_op.w13_list.127", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.0", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.1", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.2", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.3", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.4", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.5", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.6", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.7", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.8", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.9", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.10", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.11", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.12", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.13", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.14", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.15", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.16", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.17", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.18", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.19", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.20", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.21", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.22", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.23", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.24", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.25", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.26", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.27", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.28", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.29", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.30", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.31", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.32", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.33", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.34", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.35", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.36", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.37", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.38", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.39", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.40", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.41", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.42", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.43", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.44", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.45", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.46", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.47", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.48", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.49", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.50", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.51", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.52", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.53", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.54", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.55", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.56", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.57", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.58", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.59", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.60", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.61", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.62", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.63", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.64", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.65", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.66", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.67", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.68", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.69", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.70", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.71", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.72", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.73", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.74", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.75", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.76", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.77", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.78", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.79", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.80", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.81", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.82", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.83", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.84", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.85", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.86", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.87", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.88", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.89", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.90", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.91", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.92", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.93", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.94", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.95", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.96", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.97", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.98", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.99", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.100", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.101", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.102", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.103", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.104", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.105", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.106", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.107", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.108", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.109", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.110", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.111", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.112", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.113", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.114", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.115", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.116", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.117", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.118", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.119", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.120", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.121", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.122", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.123", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.124", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.125", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.126", + "language_model.model.layers.47.feed_forward.experts.moe_op.w2_list.127", + "language_model.model.layers.47.feed_forward.shared_expert.gate_up_proj", + "language_model.model.layers.47.feed_forward.shared_expert.down_proj", + "language_model.lm_head" +] \ No newline at end of file diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/maxabs_measure_g3.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/maxabs_measure_g3.json new file mode 100644 index 000000000000..60da9404943b --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/maxabs_measure_g3.json @@ -0,0 +1 @@ +{"method": "HOOKS","mode": "MEASURE","observer": "maxabs","allowlist": {"types": [], "names": []},"blocklist": {"types": [], "names": []},"quantize_weight": false,"dump_stats_path": "/eager_output/llama-4-maverick-17b-128e-instruct/g3/inc_output"} diff --git a/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/maxabs_quant_g3.json b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/maxabs_quant_g3.json new file mode 100644 index 000000000000..96a51675aa88 --- /dev/null +++ b/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/maxabs_quant_g3.json @@ -0,0 +1 @@ +{"mode": "QUANTIZE","observer": "maxabs","scale_method": "maxabs_hw","allowlist": {"types": [],"names": []},"blocklist": {"types": [],"names": []},"dump_stats_path": "./g3/inc_output"} diff --git a/.static_quant/1.22.0/README.md b/.static_quant/1.22.0/README.md new file mode 100644 index 000000000000..d6c2f652122a --- /dev/null +++ b/.static_quant/1.22.0/README.md @@ -0,0 +1,41 @@ +# Static Quantization +The below steps are given for an model Llama-4-Maverick-17B-128E-Instruct as example + +## Configuration + +1. Locate the file `maxabs_quant_g3.json` inside the `model quantization` folder. +2. Edit it and set the parameter `dump_stats_path` to the absolute path where the repository is cloned. + +Example: + +```json +"dump_stats_path": "/root/vllm-fork/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/g3/inc_output" +``` + +## Environment Variable +Export the environment variable QUANT_CONFIG before running the server. It must point to the location of maxabs_quant_g3.json. + +Example: + +```bash +export QUANT_CONFIG='/root/vllm-fork/.static_quant/1.22.0/Llama-4-Maverick-17B-128E-Instruct/maxabs_quant_g3.json' +``` + +## Run vLLM Server + +Start the vLLM server with quantization enabled: + +```bash +vllm serve meta-llama/Llama-4-Maverick-17B-128E-Instruct \ + --quantization inc \ + --kv-cache-dtype fp8_inc \ + --weights-load-device cpu \ + --tensor-parallel-size 8 \ + --max-model-len 2048 +``` + +## Notes + +1. The dump_stats_path in maxabs_quant_g3.json must be an absolute path. +2. QUANT_CONFIG must be exported before running vllm serve. +3. Adjust --tensor-parallel-size and --max-model-len according to your system resources.